From 3eec77c11a2b7f3a8a7cb7914337d32f575bc789 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 06:55:14 -0400 Subject: [PATCH] chore(cloud): add the relay fence broker, ops console, Terraform root, scripts, and 24 cloud-* workflows (#18413) Phase 6 of the relay split: the relay's deploy/operate surface moves under cloud/ with 24 cloud-* workflows gated on ORCA_CLOUD_OPERATIONS_ENABLED, the Cloud SQL rollout lease action, the relay Terraform root (dual-accept identities for both repositories), scripts, docs, CODEOWNERS, and a terraform validate job in Cloud Verify. --- .github/CODEOWNERS | 5 + .../actions/cloud-sql-rollout-lease/README.md | 152 + .../action-contract.test.mjs | 52 + .../cloud-sql-rollout-lease/action.yml | 41 + .../gcloud-access-token.mjs | 46 + .../holder-identity.mjs | 20 + .../actions/cloud-sql-rollout-lease/main.mjs | 81 + .../actions/cloud-sql-rollout-lease/post.mjs | 81 + .../actions/cloud-sql-rollout-lease/renew.mjs | 73 + .../cloud-sql-rollout-lease/runner-state.mjs | 55 + .../cloud-sql-rollout-lease/storage-lease.mjs | 240 + .../storage-lease.test.mjs | 324 + .../scripts/check-root-directory-entries.mjs | 8 +- ...cloud-bootstrap-relay-staging-capacity.yml | 676 ++ .../cloud-deploy-relay-asia-topology.yml | 245 + .../cloud-deploy-relay-fence-broker.yml | 90 + ...d-deploy-relay-production-capacity-job.yml | 822 ++ ...cloud-deploy-relay-production-capacity.yml | 353 + ...cloud-deploy-relay-production-director.yml | 267 + ...d-deploy-relay-production-multi-target.yml | 499 + ...d-deploy-relay-production-same-cap-job.yml | 645 ++ ...cloud-deploy-relay-production-same-cap.yml | 310 + .../cloud-deploy-relay-production.yml | 219 + ...oud-deploy-relay-staging-gce-candidate.yml | 109 + .../workflows/cloud-deploy-relay-staging.yml | 112 + .../cloud-monitor-relay-clock-skew.yml | 77 + .../cloud-monitor-relay-production-job.yml | 215 + .../cloud-monitor-relay-production.yml | 75 + .../cloud-operate-relay-asia-admission.yml | 514 + ...ud-operate-relay-production-rehome-job.yml | 328 + .../cloud-operate-relay-production-rehome.yml | 106 + .../workflows/cloud-power-relay-staging.yml | 101 + .../cloud-prove-relay-asia-staging.yml | 331 + .../cloud-prove-relay-staging-capacity.yml | 917 ++ .../cloud-publish-relay-production.yml | 131 + .../cloud-recover-relay-staging-c4-image.yml | 411 + ...loud-requeue-relay-staging-c4-recovery.yml | 60 + .github/workflows/cloud-verify.yml | 120 + .oxfmtrc.json | 3 +- .oxlintrc.json | 9 +- README.md | 3 + cloud/.dockerignore | 15 + cloud/.editorconfig | 10 + cloud/.gitignore | 24 + cloud/.gitleaks.toml | 15 + cloud/.node-version | 2 + cloud/.npmrc | 3 + cloud/.trufflehog-exclude-paths.txt | 1 + cloud/.trufflehog-include-paths.txt | 2 + cloud/README.md | 92 + cloud/apps/relay-fence-broker/Dockerfile | 33 + cloud/apps/relay-fence-broker/package.json | 27 + cloud/apps/relay-fence-broker/src/app.test.ts | 316 + cloud/apps/relay-fence-broker/src/app.ts | 162 + cloud/apps/relay-fence-broker/src/config.ts | 92 + .../src/fence-operation-runner.test.ts | 57 + .../src/fence-operation-runner.ts | 185 + .../relay-fence-broker/src/google-metadata.ts | 44 + cloud/apps/relay-fence-broker/src/index.ts | 10 + .../src/mutation-lease.test.ts | 164 + .../relay-fence-broker/src/mutation-lease.ts | 152 + .../relay-fence-broker/tsconfig.build.json | 10 + cloud/apps/relay-fence-broker/tsconfig.json | 5 + cloud/apps/relay-ops/README.md | 66 + cloud/apps/relay-ops/package.json | 29 + cloud/apps/relay-ops/public/app.js | 273 + cloud/apps/relay-ops/public/index.html | 115 + cloud/apps/relay-ops/public/styles.css | 165 + cloud/apps/relay-ops/src/cost-model.test.ts | 85 + cloud/apps/relay-ops/src/cost-model.ts | 115 + .../relay-ops/src/dashboard-snapshot.test.ts | 62 + .../apps/relay-ops/src/dashboard-snapshot.ts | 167 + .../relay-ops/src/environment-config.test.ts | 148 + .../apps/relay-ops/src/environment-config.ts | 125 + .../apps/relay-ops/src/gcloud-client.test.ts | 43 + cloud/apps/relay-ops/src/gcloud-client.ts | 77 + cloud/apps/relay-ops/src/github-runs.ts | 77 + .../src/incident-live-preflight-cli.test.ts | 373 + .../src/incident-live-preflight-cli.ts | 195 + .../src/incident-monitor-cli.test.ts | 454 + .../relay-ops/src/incident-monitor-cli.ts | 493 + .../src/incident-monitor-sources.test.ts | 418 + .../relay-ops/src/incident-monitor-sources.ts | 646 ++ .../relay-ops/src/incident-monitor.test.ts | 786 ++ cloud/apps/relay-ops/src/incident-monitor.ts | 800 ++ cloud/apps/relay-ops/src/incident-selector.ts | 77 + cloud/apps/relay-ops/src/index.ts | 108 + .../relay-ops/src/monitoring-snapshot.test.ts | 85 + .../apps/relay-ops/src/monitoring-snapshot.ts | 379 + .../relay-ops/src/relay-repository.test.ts | 42 + cloud/apps/relay-ops/src/relay-repository.ts | 14 + .../relay-ops/src/resource-inventory.test.ts | 150 + .../apps/relay-ops/src/resource-inventory.ts | 366 + .../relay-ops/src/staging-workflow.test.ts | 19 + cloud/apps/relay-ops/src/staging-workflow.ts | 36 + cloud/apps/relay-ops/tsconfig.build.json | 4 + cloud/apps/relay-ops/tsconfig.json | 11 + cloud/apps/relay/Dockerfile | 25 + cloud/apps/relay/package.json | 35 + .../relay/src/admin-token-verifier.test.ts | 98 + cloud/apps/relay/src/admin-token-verifier.ts | 202 + cloud/apps/relay/src/app.ts | 1850 ++++ .../src/assignment-cleanup-steps.test.ts | 67 + .../relay/src/assignment-cleanup-steps.ts | 52 + ...nment-connection-headroom-postgres.test.ts | 342 + .../assignment-connection-headroom-query.ts | 15 + .../assignment-connection-headroom.test.ts | 1007 ++ ...ment-control-supersession-postgres.test.ts | 130 + ...ignment-deployment-status-postgres.test.ts | 87 + .../src/assignment-identity-queue.test.ts | 70 + .../relay/src/assignment-identity-queue.ts | 24 + .../src/assignment-inventory-snapshot.test.ts | 107 + .../src/assignment-inventory-snapshot.ts | 170 + .../assignment-rejection-log-window.test.ts | 100 + .../src/assignment-rejection-log-window.ts | 60 + .../src/assignment-rejection-logging.test.ts | 244 + .../src/assignment-store-lock-order.test.ts | 486 + cloud/apps/relay/src/assignment-store.test.ts | 4471 +++++++++ cloud/apps/relay/src/assignment-store.ts | 8347 +++++++++++++++++ .../cell-admission-migration-registration.ts | 346 + .../relay/src/cell-admission-selector.test.ts | 580 ++ .../apps/relay/src/cell-admission-selector.ts | 505 + .../cell-admission-startup-postgres.test.ts | 83 + .../relay/src/cell-admission-startup.test.ts | 119 + .../apps/relay/src/cell-admission-startup.ts | 56 + ...ll-connection-hard-cap-consistency.test.ts | 75 + ...ell-fence-legacy-adoption-postgres.test.ts | 122 + .../relay/src/cell-heartbeat-client.test.ts | 176 + cloud/apps/relay/src/cell-heartbeat-client.ts | 123 + cloud/apps/relay/src/config.test.ts | 264 + cloud/apps/relay/src/config.ts | 348 + ...nection-reservation-debt-retention.test.ts | 130 + .../control-lease-recovery-postgres.test.ts | 223 + .../src/control-renewal-postgres.test.ts | 349 + ...ontrol-reservation-reconnect-claim.test.ts | 201 + cloud/apps/relay/src/credential-store.test.ts | 301 + cloud/apps/relay/src/credential-store.ts | 745 ++ .../src/database-postgres-timeout.test.ts | 201 + .../src/database-transient-error.test.ts | 18 + cloud/apps/relay/src/database.test.ts | 154 + cloud/apps/relay/src/database.ts | 935 ++ .../relay/src/fault-injection-test-entry.ts | 58 + .../src/google-metadata-identity-token.ts | 22 + .../relay/src/host-session-registry.test.ts | 1007 ++ cloud/apps/relay/src/host-session-registry.ts | 1226 +++ cloud/apps/relay/src/index.ts | 131 + .../src/migration-recovery-postgres.test.ts | 1376 +++ .../apps/relay/src/observed-relay-database.ts | 46 + .../src/postgres-drain-send-locking.test.ts | 215 + .../src/postgres-idle-client-error.test.ts | 22 + .../postgres-maintenance-sweep-plans.test.ts | 61 + .../relay/src/postgres-pool-pressure.test.ts | 48 + .../apps/relay/src/postgres-pool-pressure.ts | 93 + ...stgres-schema-concurrency-postgres.test.ts | 55 + .../apps/relay/src/postgres-schema-startup.ts | 83 + .../src/postgres-transaction-recovery.test.ts | 1446 +++ .../src/public-assignment-admission.test.ts | 334 + .../relay/src/public-assignment-admission.ts | 239 + .../public-assignment-circuit-breaker.test.ts | 55 + .../src/public-assignment-overload.test.ts | 236 + .../public-assignment-reconnect-lane.test.ts | 202 + .../relay/src/regional-host-drain-app.test.ts | 536 ++ .../src/regional-rehome-postgres.test.ts | 552 ++ .../relay/src/regional-rehome-safety.test.ts | 109 + .../apps/relay/src/regional-rehome-safety.ts | 86 + .../relay/src/regional-rehome-store.test.ts | 1399 +++ .../regional-rehome-target-selection.test.ts | 163 + .../relay/src/regional-rehome-trust-probe.ts | 107 + .../relay/src/regional-rehome-worker.test.ts | 227 + .../apps/relay/src/regional-rehome-worker.ts | 122 + .../src/registered-migration-abandonment.ts | 81 + ...tered-migration-inventory-postgres.test.ts | 139 + .../registered-migration-inventory.test.ts | 117 + .../src/registered-migration-inventory.ts | 104 + .../src/relay-background-operation.test.ts | 44 + .../relay/src/relay-background-operation.ts | 28 + ...relay-connection-hard-cap.blackbox.test.ts | 301 + .../relay/src/relay-connection-ledger.test.ts | 172 + .../apps/relay/src/relay-connection-ledger.ts | 229 + ...relay-first-frame-failure.blackbox.test.ts | 124 + cloud/apps/relay/src/relay-host-log-digest.ts | 7 + .../relay-host-proof-failure.blackbox.test.ts | 153 + .../relay/src/relay-observability.test.ts | 246 + cloud/apps/relay/src/relay-observability.ts | 336 + cloud/apps/relay/src/relay-readiness.test.ts | 109 + cloud/apps/relay/src/relay-readiness.ts | 89 + cloud/apps/relay/src/relay-region-app.test.ts | 155 + .../relay/src/relay-region-placement.test.ts | 196 + cloud/apps/relay/src/relay-server.ts | 533 ++ cloud/apps/relay/src/relay-token-verifier.ts | 35 + .../relay/src/relay-websocket-close.test.ts | 44 + cloud/apps/relay/src/relay-websocket-close.ts | 22 + cloud/apps/relay/src/relay.blackbox.test.ts | 2622 ++++++ cloud/apps/relay/src/splice-forwarder.test.ts | 135 + cloud/apps/relay/src/splice-forwarder.ts | 158 + .../src/staging-asia-proof-admission.test.ts | 37 + .../relay/src/staging-asia-proof-admission.ts | 30 + cloud/apps/relay/tsconfig.build.json | 10 + cloud/apps/relay/tsconfig.json | 5 + cloud/apps/relay/vitest.config.ts | 44 + .../production-cloud-sql-app-consumers.json | 15 + .../terraform-root-partition/families.json | 267 + .../capture-terraform-plan-baseline.mjs | 83 + .../capture-terraform-plan-baseline.test.mjs | 15 + ...ify-relay-production-capacity-director.mjs | 109 + ...elay-production-capacity-director.test.mjs | 93 + .../classify-relay-staging-bootstrap.mjs | 47 + .../classify-relay-staging-bootstrap.test.mjs | 60 + .../scripts/cloud-sql-rollout-lock-census.mjs | 305 + cloud/dev/scripts/deploy-relay-blue-green.mjs | 1132 +++ .../scripts/deploy-relay-blue-green.test.mjs | 814 ++ .../scripts/deploy-relay-gce-candidate.mjs | 871 ++ .../deploy-relay-gce-candidate.test.mjs | 889 ++ .../scripts/deploy-relay-gce-multi-target.mjs | 3135 +++++++ .../deploy-relay-gce-multi-target.test.mjs | 3320 +++++++ cloud/dev/scripts/github-smoke-token.mjs | 133 + cloud/dev/scripts/github-smoke-token.test.mjs | 111 + cloud/dev/scripts/infra.mjs | 146 + cloud/dev/scripts/infra.test.mjs | 77 + cloud/dev/scripts/load-relay-controls.mjs | 570 ++ .../scripts/operate-relay-asia-admission.mjs | 440 + .../operate-relay-asia-admission.test.mjs | 512 + .../scripts/operate-relay-regional-rehome.mjs | 312 + .../operate-relay-regional-rehome.test.mjs | 265 + cloud/dev/scripts/power-staging-relay.mjs | 487 + .../dev/scripts/power-staging-relay.test.mjs | 359 + .../prepare-relay-asia-director-cells.mjs | 78 + ...prepare-relay-asia-director-cells.test.mjs | 60 + .../prepare-relay-asia-topology-input.mjs | 113 + ...prepare-relay-asia-topology-input.test.mjs | 87 + .../scripts/prepare-relay-capacity-canary.mjs | 183 + .../prepare-relay-capacity-canary.test.mjs | 300 + ...epare-relay-production-capacity-canary.mjs | 116 + ...-relay-production-capacity-canary.test.mjs | 173 + .../scripts/probe-relay-legacy-admission.mjs | 73 + .../probe-relay-legacy-admission.test.mjs | 89 + .../dev/scripts/probe-relay-rehome-trust.mjs | 80 + .../scripts/probe-relay-rehome-trust.test.mjs | 70 + ...ion-cell-image-digest-consistency.test.mjs | 85 + ...production-cloud-sql-rollout-lock.test.mjs | 210 + ...ead-relay-production-capacity-identity.mjs | 31 + ...elay-production-capacity-identity.test.mjs | 44 + ...lay-serving-regional-placement-version.mjs | 106 + ...erving-regional-placement-version.test.mjs | 138 + .../dev/scripts/relay-admission-selector.mjs | 277 + .../scripts/relay-admission-selector.test.mjs | 232 + .../relay-asia-admission-workflow.test.mjs | 323 + .../scripts/relay-asia-cloud-sql-metrics.mjs | 33 + .../scripts/relay-asia-rollout-evidence.mjs | 544 ++ .../relay-asia-rollout-evidence.test.mjs | 357 + .../relay-asia-topology-workflow.test.mjs | 124 + .../relay-cloud-sql-connection-budget.mjs | 148 + ...relay-cloud-sql-connection-budget.test.mjs | 110 + .../dev/scripts/relay-gce-terraform-fence.mjs | 1387 +++ .../relay-gce-terraform-fence.test.mjs | 1522 +++ .../scripts/relay-load-connection-failure.mjs | 37 + .../relay-load-connection-failure.test.mjs | 57 + cloud/dev/scripts/relay-load-control-peer.mjs | 891 ++ .../scripts/relay-load-control-peer.test.mjs | 903 ++ .../relay-load-director-capacity-gate.mjs | 147 + ...relay-load-director-capacity-gate.test.mjs | 260 + cloud/dev/scripts/relay-load-model.mjs | 76 + cloud/dev/scripts/relay-load-model.test.mjs | 25 + .../dev/scripts/relay-load-phase-barrier.mjs | 37 + .../scripts/relay-load-phase-barrier.test.mjs | 65 + .../scripts/relay-load-placement-boundary.mjs | 29 + .../relay-load-placement-boundary.test.mjs | 53 + cloud/dev/scripts/relay-load-profile.mjs | 404 + cloud/dev/scripts/relay-load-profile.test.mjs | 388 + .../scripts/relay-load-reader-evidence.mjs | 51 + .../relay-load-reader-evidence.test.mjs | 76 + .../scripts/relay-load-rebind-boundary.mjs | 59 + .../relay-load-rebind-boundary.test.mjs | 164 + .../scripts/relay-load-region-behavior.mjs | 35 + .../relay-load-region-behavior.test.mjs | 47 + .../relay-load-request-unit-boundary.mjs | 43 + .../relay-load-request-unit-boundary.test.mjs | 52 + .../dev/scripts/relay-load-run-lifecycle.mjs | 25 + .../scripts/relay-load-run-lifecycle.test.mjs | 68 + cloud/dev/scripts/relay-monitor-evidence.mjs | 355 + .../scripts/relay-monitor-evidence.test.mjs | 515 + .../relay-production-capacity-wave.mjs | 172 + .../relay-production-capacity-wave.test.mjs | 129 + ...elay-production-capacity-workflow.test.mjs | 440 + ...ay-production-identity-boundaries.test.mjs | 169 + .../relay-production-same-cap-wave.mjs | 161 + .../relay-production-same-cap-wave.test.mjs | 106 + .../relay-public-workflow-contract.test.mjs | 82 + .../dev/scripts/relay-recovery-wave-gate.mjs | 517 + .../scripts/relay-recovery-wave-gate.test.mjs | 214 + .../relay-region-observation-evidence.mjs | 152 + ...relay-region-observation-evidence.test.mjs | 53 + .../relay-regional-rehome-workflow.test.mjs | 193 + .../relay-rehome-aggregate-evidence.mjs | 65 + .../relay-rehome-aggregate-evidence.test.mjs | 38 + cloud/dev/scripts/relay-repository.mjs | 29 + cloud/dev/scripts/relay-repository.test.mjs | 36 + ...relay-staging-c4-refresh-workflow.test.mjs | 159 + .../relay-staging-capacity-identity.test.mjs | 269 + .../relay-staging-deploy-identity.test.mjs | 228 + .../render-workload-identity-conditions.mjs | 535 ++ cloud/dev/scripts/run-relay-load-model.mjs | 8 + .../scripts/run-relay-recovery-wave-gate.mjs | 19 + .../sanitize-relay-asia-admission-result.mjs | 98 + ...itize-relay-asia-admission-result.test.mjs | 120 + cloud/dev/scripts/smoke-relay.mjs | 211 + .../dev/scripts/staging-relay-apply-guard.mjs | 51 + .../staging-relay-apply-guard.test.mjs | 30 + .../dev/scripts/terraform-root-partition.mjs | 131 + .../scripts/terraform-root-partition.test.mjs | 117 + .../validate-relay-asia-topology-plan.mjs | 295 + ...validate-relay-asia-topology-plan.test.mjs | 223 + .../scripts/validate-relay-capacity-plan.mjs | 519 + .../validate-relay-capacity-plan.test.mjs | 646 ++ .../verify-relay-capacity-transition.mjs | 473 + .../verify-relay-capacity-transition.test.mjs | 1096 +++ .../scripts/verify-relay-legacy-bootstrap.mjs | 292 + .../verify-relay-legacy-bootstrap.test.mjs | 395 + ...oad-identity-attribute-conditions.test.mjs | 157 + cloud/docs/orca-relay-capacity-testing.md | 259 + cloud/docs/orca-relay-operations.md | 481 + cloud/docs/relay-incident-monitor.md | 152 + cloud/docs/relay-terraform-fencing-plan.md | 149 + cloud/docs/relay-workflows.md | 402 + .../staging-relay-deploy-identity-rollout.md | 177 + cloud/infra/terraform/.terraform.lock.hcl | 67 + cloud/infra/terraform/README.md | 409 + cloud/infra/terraform/backend/production.hcl | 3 + cloud/infra/terraform/backend/staging.hcl | 3 + .../terraform/environments/production.tfvars | 418 + .../terraform/environments/staging.tfvars | 91 + cloud/infra/terraform/outputs.tf | 191 + cloud/infra/terraform/relay-asia-proof-iam.tf | 79 + .../terraform/relay-asia-topology-iam.tf | 207 + cloud/infra/terraform/relay-database.tf | 54 + cloud/infra/terraform/relay-dns.tf | 47 + cloud/infra/terraform/relay-fence-broker.tf | 243 + cloud/infra/terraform/relay-gce-cells.tf | 393 + cloud/infra/terraform/relay-gce-foundation.tf | 200 + .../terraform/relay-gce-startup.sh.tftpl | 136 + cloud/infra/terraform/relay-github-actions.tf | 669 ++ .../terraform/relay-github-workflow-trust.tf | 38 + cloud/infra/terraform/relay-observability.tf | 525 ++ cloud/infra/terraform/relay-shared.tf | 90 + .../terraform/relay-staging-deploy-iam.tf | 170 + cloud/infra/terraform/relay.tf | 577 ++ cloud/infra/terraform/variables.tf | 468 + cloud/infra/terraform/versions.tf | 27 + cloud/package.json | 33 + cloud/packages/relay-contract/package.json | 23 + .../relay-contract/src/admission-budgets.ts | 76 + .../src/assignment-invariants.ts | 56 + .../relay-contract/src/close-codes.ts | 10 + .../relay-contract/src/contract.test.ts | 347 + .../relay-contract/src/control-continuity.ts | 43 + .../relay-contract/src/control-messages.ts | 119 + .../relay-contract/src/credential-messages.ts | 105 + .../relay-contract/src/director-messages.ts | 75 + .../src/host-proof-transcript.ts | 103 + cloud/packages/relay-contract/src/index.ts | 14 + .../src/persistence-invariants.test.ts | 156 + .../src/persistence-invariants.ts | 172 + .../relay-contract/src/protocol-limits.ts | 23 + .../relay-contract/src/relay-regions.ts | 62 + .../src/resume-confirmation-contract.ts | 23 + .../src/splice-state-machine.ts | 34 + .../relay-contract/src/wire-scalars.ts | 20 + .../relay-contract/tsconfig.build.json | 11 + cloud/packages/relay-contract/tsconfig.json | 5 + cloud/pnpm-lock.yaml | 1336 +++ cloud/pnpm-workspace.yaml | 4 + cloud/tsconfig.base.json | 19 + config/oxlint-react-doctor.json | 2 +- config/scripts/check-changed-code-quality.mjs | 17 +- .../check-changed-code-quality.test.mjs | 6 + .../check-root-directory-entries.test.mjs | 9 + config/scripts/pr-code-change-scope.mjs | 2 + config/scripts/pr-code-change-scope.test.mjs | 11 + package.json | 4 +- 379 files changed, 102741 insertions(+), 8 deletions(-) create mode 100644 .github/actions/cloud-sql-rollout-lease/README.md create mode 100644 .github/actions/cloud-sql-rollout-lease/action-contract.test.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/action.yml create mode 100644 .github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/holder-identity.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/main.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/post.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/renew.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/runner-state.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/storage-lease.mjs create mode 100644 .github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs create mode 100644 .github/workflows/cloud-bootstrap-relay-staging-capacity.yml create mode 100644 .github/workflows/cloud-deploy-relay-asia-topology.yml create mode 100644 .github/workflows/cloud-deploy-relay-fence-broker.yml create mode 100644 .github/workflows/cloud-deploy-relay-production-capacity-job.yml create mode 100644 .github/workflows/cloud-deploy-relay-production-capacity.yml create mode 100644 .github/workflows/cloud-deploy-relay-production-director.yml create mode 100644 .github/workflows/cloud-deploy-relay-production-multi-target.yml create mode 100644 .github/workflows/cloud-deploy-relay-production-same-cap-job.yml create mode 100644 .github/workflows/cloud-deploy-relay-production-same-cap.yml create mode 100644 .github/workflows/cloud-deploy-relay-production.yml create mode 100644 .github/workflows/cloud-deploy-relay-staging-gce-candidate.yml create mode 100644 .github/workflows/cloud-deploy-relay-staging.yml create mode 100644 .github/workflows/cloud-monitor-relay-clock-skew.yml create mode 100644 .github/workflows/cloud-monitor-relay-production-job.yml create mode 100644 .github/workflows/cloud-monitor-relay-production.yml create mode 100644 .github/workflows/cloud-operate-relay-asia-admission.yml create mode 100644 .github/workflows/cloud-operate-relay-production-rehome-job.yml create mode 100644 .github/workflows/cloud-operate-relay-production-rehome.yml create mode 100644 .github/workflows/cloud-power-relay-staging.yml create mode 100644 .github/workflows/cloud-prove-relay-asia-staging.yml create mode 100644 .github/workflows/cloud-prove-relay-staging-capacity.yml create mode 100644 .github/workflows/cloud-publish-relay-production.yml create mode 100644 .github/workflows/cloud-recover-relay-staging-c4-image.yml create mode 100644 .github/workflows/cloud-requeue-relay-staging-c4-recovery.yml create mode 100644 .github/workflows/cloud-verify.yml create mode 100644 cloud/.dockerignore create mode 100644 cloud/.editorconfig create mode 100644 cloud/.gitignore create mode 100644 cloud/.gitleaks.toml create mode 100644 cloud/.node-version create mode 100644 cloud/.npmrc create mode 100644 cloud/.trufflehog-exclude-paths.txt create mode 100644 cloud/.trufflehog-include-paths.txt create mode 100644 cloud/README.md create mode 100644 cloud/apps/relay-fence-broker/Dockerfile create mode 100644 cloud/apps/relay-fence-broker/package.json create mode 100644 cloud/apps/relay-fence-broker/src/app.test.ts create mode 100644 cloud/apps/relay-fence-broker/src/app.ts create mode 100644 cloud/apps/relay-fence-broker/src/config.ts create mode 100644 cloud/apps/relay-fence-broker/src/fence-operation-runner.test.ts create mode 100644 cloud/apps/relay-fence-broker/src/fence-operation-runner.ts create mode 100644 cloud/apps/relay-fence-broker/src/google-metadata.ts create mode 100644 cloud/apps/relay-fence-broker/src/index.ts create mode 100644 cloud/apps/relay-fence-broker/src/mutation-lease.test.ts create mode 100644 cloud/apps/relay-fence-broker/src/mutation-lease.ts create mode 100644 cloud/apps/relay-fence-broker/tsconfig.build.json create mode 100644 cloud/apps/relay-fence-broker/tsconfig.json create mode 100644 cloud/apps/relay-ops/README.md create mode 100644 cloud/apps/relay-ops/package.json create mode 100644 cloud/apps/relay-ops/public/app.js create mode 100644 cloud/apps/relay-ops/public/index.html create mode 100644 cloud/apps/relay-ops/public/styles.css create mode 100644 cloud/apps/relay-ops/src/cost-model.test.ts create mode 100644 cloud/apps/relay-ops/src/cost-model.ts create mode 100644 cloud/apps/relay-ops/src/dashboard-snapshot.test.ts create mode 100644 cloud/apps/relay-ops/src/dashboard-snapshot.ts create mode 100644 cloud/apps/relay-ops/src/environment-config.test.ts create mode 100644 cloud/apps/relay-ops/src/environment-config.ts create mode 100644 cloud/apps/relay-ops/src/gcloud-client.test.ts create mode 100644 cloud/apps/relay-ops/src/gcloud-client.ts create mode 100644 cloud/apps/relay-ops/src/github-runs.ts create mode 100644 cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts create mode 100644 cloud/apps/relay-ops/src/incident-live-preflight-cli.ts create mode 100644 cloud/apps/relay-ops/src/incident-monitor-cli.test.ts create mode 100644 cloud/apps/relay-ops/src/incident-monitor-cli.ts create mode 100644 cloud/apps/relay-ops/src/incident-monitor-sources.test.ts create mode 100644 cloud/apps/relay-ops/src/incident-monitor-sources.ts create mode 100644 cloud/apps/relay-ops/src/incident-monitor.test.ts create mode 100644 cloud/apps/relay-ops/src/incident-monitor.ts create mode 100644 cloud/apps/relay-ops/src/incident-selector.ts create mode 100644 cloud/apps/relay-ops/src/index.ts create mode 100644 cloud/apps/relay-ops/src/monitoring-snapshot.test.ts create mode 100644 cloud/apps/relay-ops/src/monitoring-snapshot.ts create mode 100644 cloud/apps/relay-ops/src/relay-repository.test.ts create mode 100644 cloud/apps/relay-ops/src/relay-repository.ts create mode 100644 cloud/apps/relay-ops/src/resource-inventory.test.ts create mode 100644 cloud/apps/relay-ops/src/resource-inventory.ts create mode 100644 cloud/apps/relay-ops/src/staging-workflow.test.ts create mode 100644 cloud/apps/relay-ops/src/staging-workflow.ts create mode 100644 cloud/apps/relay-ops/tsconfig.build.json create mode 100644 cloud/apps/relay-ops/tsconfig.json create mode 100644 cloud/apps/relay/Dockerfile create mode 100644 cloud/apps/relay/package.json create mode 100644 cloud/apps/relay/src/admin-token-verifier.test.ts create mode 100644 cloud/apps/relay/src/admin-token-verifier.ts create mode 100644 cloud/apps/relay/src/app.ts create mode 100644 cloud/apps/relay/src/assignment-cleanup-steps.test.ts create mode 100644 cloud/apps/relay/src/assignment-cleanup-steps.ts create mode 100644 cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts create mode 100644 cloud/apps/relay/src/assignment-connection-headroom-query.ts create mode 100644 cloud/apps/relay/src/assignment-connection-headroom.test.ts create mode 100644 cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts create mode 100644 cloud/apps/relay/src/assignment-deployment-status-postgres.test.ts create mode 100644 cloud/apps/relay/src/assignment-identity-queue.test.ts create mode 100644 cloud/apps/relay/src/assignment-identity-queue.ts create mode 100644 cloud/apps/relay/src/assignment-inventory-snapshot.test.ts create mode 100644 cloud/apps/relay/src/assignment-inventory-snapshot.ts create mode 100644 cloud/apps/relay/src/assignment-rejection-log-window.test.ts create mode 100644 cloud/apps/relay/src/assignment-rejection-log-window.ts create mode 100644 cloud/apps/relay/src/assignment-rejection-logging.test.ts create mode 100644 cloud/apps/relay/src/assignment-store-lock-order.test.ts create mode 100644 cloud/apps/relay/src/assignment-store.test.ts create mode 100644 cloud/apps/relay/src/assignment-store.ts create mode 100644 cloud/apps/relay/src/cell-admission-migration-registration.ts create mode 100644 cloud/apps/relay/src/cell-admission-selector.test.ts create mode 100644 cloud/apps/relay/src/cell-admission-selector.ts create mode 100644 cloud/apps/relay/src/cell-admission-startup-postgres.test.ts create mode 100644 cloud/apps/relay/src/cell-admission-startup.test.ts create mode 100644 cloud/apps/relay/src/cell-admission-startup.ts create mode 100644 cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts create mode 100644 cloud/apps/relay/src/cell-fence-legacy-adoption-postgres.test.ts create mode 100644 cloud/apps/relay/src/cell-heartbeat-client.test.ts create mode 100644 cloud/apps/relay/src/cell-heartbeat-client.ts create mode 100644 cloud/apps/relay/src/config.test.ts create mode 100644 cloud/apps/relay/src/config.ts create mode 100644 cloud/apps/relay/src/connection-reservation-debt-retention.test.ts create mode 100644 cloud/apps/relay/src/control-lease-recovery-postgres.test.ts create mode 100644 cloud/apps/relay/src/control-renewal-postgres.test.ts create mode 100644 cloud/apps/relay/src/control-reservation-reconnect-claim.test.ts create mode 100644 cloud/apps/relay/src/credential-store.test.ts create mode 100644 cloud/apps/relay/src/credential-store.ts create mode 100644 cloud/apps/relay/src/database-postgres-timeout.test.ts create mode 100644 cloud/apps/relay/src/database-transient-error.test.ts create mode 100644 cloud/apps/relay/src/database.test.ts create mode 100644 cloud/apps/relay/src/database.ts create mode 100644 cloud/apps/relay/src/fault-injection-test-entry.ts create mode 100644 cloud/apps/relay/src/google-metadata-identity-token.ts create mode 100644 cloud/apps/relay/src/host-session-registry.test.ts create mode 100644 cloud/apps/relay/src/host-session-registry.ts create mode 100644 cloud/apps/relay/src/index.ts create mode 100644 cloud/apps/relay/src/migration-recovery-postgres.test.ts create mode 100644 cloud/apps/relay/src/observed-relay-database.ts create mode 100644 cloud/apps/relay/src/postgres-drain-send-locking.test.ts create mode 100644 cloud/apps/relay/src/postgres-idle-client-error.test.ts create mode 100644 cloud/apps/relay/src/postgres-maintenance-sweep-plans.test.ts create mode 100644 cloud/apps/relay/src/postgres-pool-pressure.test.ts create mode 100644 cloud/apps/relay/src/postgres-pool-pressure.ts create mode 100644 cloud/apps/relay/src/postgres-schema-concurrency-postgres.test.ts create mode 100644 cloud/apps/relay/src/postgres-schema-startup.ts create mode 100644 cloud/apps/relay/src/postgres-transaction-recovery.test.ts create mode 100644 cloud/apps/relay/src/public-assignment-admission.test.ts create mode 100644 cloud/apps/relay/src/public-assignment-admission.ts create mode 100644 cloud/apps/relay/src/public-assignment-circuit-breaker.test.ts create mode 100644 cloud/apps/relay/src/public-assignment-overload.test.ts create mode 100644 cloud/apps/relay/src/public-assignment-reconnect-lane.test.ts create mode 100644 cloud/apps/relay/src/regional-host-drain-app.test.ts create mode 100644 cloud/apps/relay/src/regional-rehome-postgres.test.ts create mode 100644 cloud/apps/relay/src/regional-rehome-safety.test.ts create mode 100644 cloud/apps/relay/src/regional-rehome-safety.ts create mode 100644 cloud/apps/relay/src/regional-rehome-store.test.ts create mode 100644 cloud/apps/relay/src/regional-rehome-target-selection.test.ts create mode 100644 cloud/apps/relay/src/regional-rehome-trust-probe.ts create mode 100644 cloud/apps/relay/src/regional-rehome-worker.test.ts create mode 100644 cloud/apps/relay/src/regional-rehome-worker.ts create mode 100644 cloud/apps/relay/src/registered-migration-abandonment.ts create mode 100644 cloud/apps/relay/src/registered-migration-inventory-postgres.test.ts create mode 100644 cloud/apps/relay/src/registered-migration-inventory.test.ts create mode 100644 cloud/apps/relay/src/registered-migration-inventory.ts create mode 100644 cloud/apps/relay/src/relay-background-operation.test.ts create mode 100644 cloud/apps/relay/src/relay-background-operation.ts create mode 100644 cloud/apps/relay/src/relay-connection-hard-cap.blackbox.test.ts create mode 100644 cloud/apps/relay/src/relay-connection-ledger.test.ts create mode 100644 cloud/apps/relay/src/relay-connection-ledger.ts create mode 100644 cloud/apps/relay/src/relay-first-frame-failure.blackbox.test.ts create mode 100644 cloud/apps/relay/src/relay-host-log-digest.ts create mode 100644 cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts create mode 100644 cloud/apps/relay/src/relay-observability.test.ts create mode 100644 cloud/apps/relay/src/relay-observability.ts create mode 100644 cloud/apps/relay/src/relay-readiness.test.ts create mode 100644 cloud/apps/relay/src/relay-readiness.ts create mode 100644 cloud/apps/relay/src/relay-region-app.test.ts create mode 100644 cloud/apps/relay/src/relay-region-placement.test.ts create mode 100644 cloud/apps/relay/src/relay-server.ts create mode 100644 cloud/apps/relay/src/relay-token-verifier.ts create mode 100644 cloud/apps/relay/src/relay-websocket-close.test.ts create mode 100644 cloud/apps/relay/src/relay-websocket-close.ts create mode 100644 cloud/apps/relay/src/relay.blackbox.test.ts create mode 100644 cloud/apps/relay/src/splice-forwarder.test.ts create mode 100644 cloud/apps/relay/src/splice-forwarder.ts create mode 100644 cloud/apps/relay/src/staging-asia-proof-admission.test.ts create mode 100644 cloud/apps/relay/src/staging-asia-proof-admission.ts create mode 100644 cloud/apps/relay/tsconfig.build.json create mode 100644 cloud/apps/relay/tsconfig.json create mode 100644 cloud/apps/relay/vitest.config.ts create mode 100644 cloud/dev/contracts/production-cloud-sql-app-consumers.json create mode 100644 cloud/dev/fixtures/terraform-root-partition/families.json create mode 100644 cloud/dev/scripts/capture-terraform-plan-baseline.mjs create mode 100644 cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs create mode 100644 cloud/dev/scripts/classify-relay-production-capacity-director.mjs create mode 100644 cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs create mode 100644 cloud/dev/scripts/classify-relay-staging-bootstrap.mjs create mode 100644 cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs create mode 100644 cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs create mode 100644 cloud/dev/scripts/deploy-relay-blue-green.mjs create mode 100644 cloud/dev/scripts/deploy-relay-blue-green.test.mjs create mode 100644 cloud/dev/scripts/deploy-relay-gce-candidate.mjs create mode 100644 cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs create mode 100644 cloud/dev/scripts/deploy-relay-gce-multi-target.mjs create mode 100644 cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs create mode 100644 cloud/dev/scripts/github-smoke-token.mjs create mode 100644 cloud/dev/scripts/github-smoke-token.test.mjs create mode 100644 cloud/dev/scripts/infra.mjs create mode 100644 cloud/dev/scripts/infra.test.mjs create mode 100644 cloud/dev/scripts/load-relay-controls.mjs create mode 100644 cloud/dev/scripts/operate-relay-asia-admission.mjs create mode 100644 cloud/dev/scripts/operate-relay-asia-admission.test.mjs create mode 100644 cloud/dev/scripts/operate-relay-regional-rehome.mjs create mode 100644 cloud/dev/scripts/operate-relay-regional-rehome.test.mjs create mode 100644 cloud/dev/scripts/power-staging-relay.mjs create mode 100644 cloud/dev/scripts/power-staging-relay.test.mjs create mode 100644 cloud/dev/scripts/prepare-relay-asia-director-cells.mjs create mode 100644 cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs create mode 100644 cloud/dev/scripts/prepare-relay-asia-topology-input.mjs create mode 100644 cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs create mode 100644 cloud/dev/scripts/prepare-relay-capacity-canary.mjs create mode 100644 cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs create mode 100644 cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs create mode 100644 cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs create mode 100644 cloud/dev/scripts/probe-relay-legacy-admission.mjs create mode 100644 cloud/dev/scripts/probe-relay-legacy-admission.test.mjs create mode 100644 cloud/dev/scripts/probe-relay-rehome-trust.mjs create mode 100644 cloud/dev/scripts/probe-relay-rehome-trust.test.mjs create mode 100644 cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs create mode 100644 cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs create mode 100644 cloud/dev/scripts/read-relay-production-capacity-identity.mjs create mode 100644 cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs create mode 100644 cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs create mode 100644 cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs create mode 100644 cloud/dev/scripts/relay-admission-selector.mjs create mode 100644 cloud/dev/scripts/relay-admission-selector.test.mjs create mode 100644 cloud/dev/scripts/relay-asia-admission-workflow.test.mjs create mode 100644 cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs create mode 100644 cloud/dev/scripts/relay-asia-rollout-evidence.mjs create mode 100644 cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs create mode 100644 cloud/dev/scripts/relay-asia-topology-workflow.test.mjs create mode 100644 cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs create mode 100644 cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs create mode 100644 cloud/dev/scripts/relay-gce-terraform-fence.mjs create mode 100644 cloud/dev/scripts/relay-gce-terraform-fence.test.mjs create mode 100644 cloud/dev/scripts/relay-load-connection-failure.mjs create mode 100644 cloud/dev/scripts/relay-load-connection-failure.test.mjs create mode 100644 cloud/dev/scripts/relay-load-control-peer.mjs create mode 100644 cloud/dev/scripts/relay-load-control-peer.test.mjs create mode 100644 cloud/dev/scripts/relay-load-director-capacity-gate.mjs create mode 100644 cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs create mode 100644 cloud/dev/scripts/relay-load-model.mjs create mode 100644 cloud/dev/scripts/relay-load-model.test.mjs create mode 100644 cloud/dev/scripts/relay-load-phase-barrier.mjs create mode 100644 cloud/dev/scripts/relay-load-phase-barrier.test.mjs create mode 100644 cloud/dev/scripts/relay-load-placement-boundary.mjs create mode 100644 cloud/dev/scripts/relay-load-placement-boundary.test.mjs create mode 100644 cloud/dev/scripts/relay-load-profile.mjs create mode 100644 cloud/dev/scripts/relay-load-profile.test.mjs create mode 100644 cloud/dev/scripts/relay-load-reader-evidence.mjs create mode 100644 cloud/dev/scripts/relay-load-reader-evidence.test.mjs create mode 100644 cloud/dev/scripts/relay-load-rebind-boundary.mjs create mode 100644 cloud/dev/scripts/relay-load-rebind-boundary.test.mjs create mode 100644 cloud/dev/scripts/relay-load-region-behavior.mjs create mode 100644 cloud/dev/scripts/relay-load-region-behavior.test.mjs create mode 100644 cloud/dev/scripts/relay-load-request-unit-boundary.mjs create mode 100644 cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs create mode 100644 cloud/dev/scripts/relay-load-run-lifecycle.mjs create mode 100644 cloud/dev/scripts/relay-load-run-lifecycle.test.mjs create mode 100644 cloud/dev/scripts/relay-monitor-evidence.mjs create mode 100644 cloud/dev/scripts/relay-monitor-evidence.test.mjs create mode 100644 cloud/dev/scripts/relay-production-capacity-wave.mjs create mode 100644 cloud/dev/scripts/relay-production-capacity-wave.test.mjs create mode 100644 cloud/dev/scripts/relay-production-capacity-workflow.test.mjs create mode 100644 cloud/dev/scripts/relay-production-identity-boundaries.test.mjs create mode 100644 cloud/dev/scripts/relay-production-same-cap-wave.mjs create mode 100644 cloud/dev/scripts/relay-production-same-cap-wave.test.mjs create mode 100644 cloud/dev/scripts/relay-public-workflow-contract.test.mjs create mode 100644 cloud/dev/scripts/relay-recovery-wave-gate.mjs create mode 100644 cloud/dev/scripts/relay-recovery-wave-gate.test.mjs create mode 100644 cloud/dev/scripts/relay-region-observation-evidence.mjs create mode 100644 cloud/dev/scripts/relay-region-observation-evidence.test.mjs create mode 100644 cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs create mode 100644 cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs create mode 100644 cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs create mode 100644 cloud/dev/scripts/relay-repository.mjs create mode 100644 cloud/dev/scripts/relay-repository.test.mjs create mode 100644 cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs create mode 100644 cloud/dev/scripts/relay-staging-capacity-identity.test.mjs create mode 100644 cloud/dev/scripts/relay-staging-deploy-identity.test.mjs create mode 100644 cloud/dev/scripts/render-workload-identity-conditions.mjs create mode 100644 cloud/dev/scripts/run-relay-load-model.mjs create mode 100644 cloud/dev/scripts/run-relay-recovery-wave-gate.mjs create mode 100644 cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs create mode 100644 cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs create mode 100644 cloud/dev/scripts/smoke-relay.mjs create mode 100644 cloud/dev/scripts/staging-relay-apply-guard.mjs create mode 100644 cloud/dev/scripts/staging-relay-apply-guard.test.mjs create mode 100644 cloud/dev/scripts/terraform-root-partition.mjs create mode 100644 cloud/dev/scripts/terraform-root-partition.test.mjs create mode 100644 cloud/dev/scripts/validate-relay-asia-topology-plan.mjs create mode 100644 cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs create mode 100644 cloud/dev/scripts/validate-relay-capacity-plan.mjs create mode 100644 cloud/dev/scripts/validate-relay-capacity-plan.test.mjs create mode 100644 cloud/dev/scripts/verify-relay-capacity-transition.mjs create mode 100644 cloud/dev/scripts/verify-relay-capacity-transition.test.mjs create mode 100644 cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs create mode 100644 cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs create mode 100644 cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs create mode 100644 cloud/docs/orca-relay-capacity-testing.md create mode 100644 cloud/docs/orca-relay-operations.md create mode 100644 cloud/docs/relay-incident-monitor.md create mode 100644 cloud/docs/relay-terraform-fencing-plan.md create mode 100644 cloud/docs/relay-workflows.md create mode 100644 cloud/docs/staging-relay-deploy-identity-rollout.md create mode 100644 cloud/infra/terraform/.terraform.lock.hcl create mode 100644 cloud/infra/terraform/README.md create mode 100644 cloud/infra/terraform/backend/production.hcl create mode 100644 cloud/infra/terraform/backend/staging.hcl create mode 100644 cloud/infra/terraform/environments/production.tfvars create mode 100644 cloud/infra/terraform/environments/staging.tfvars create mode 100644 cloud/infra/terraform/outputs.tf create mode 100644 cloud/infra/terraform/relay-asia-proof-iam.tf create mode 100644 cloud/infra/terraform/relay-asia-topology-iam.tf create mode 100644 cloud/infra/terraform/relay-database.tf create mode 100644 cloud/infra/terraform/relay-dns.tf create mode 100644 cloud/infra/terraform/relay-fence-broker.tf create mode 100644 cloud/infra/terraform/relay-gce-cells.tf create mode 100644 cloud/infra/terraform/relay-gce-foundation.tf create mode 100644 cloud/infra/terraform/relay-gce-startup.sh.tftpl create mode 100644 cloud/infra/terraform/relay-github-actions.tf create mode 100644 cloud/infra/terraform/relay-github-workflow-trust.tf create mode 100644 cloud/infra/terraform/relay-observability.tf create mode 100644 cloud/infra/terraform/relay-shared.tf create mode 100644 cloud/infra/terraform/relay-staging-deploy-iam.tf create mode 100644 cloud/infra/terraform/relay.tf create mode 100644 cloud/infra/terraform/variables.tf create mode 100644 cloud/infra/terraform/versions.tf create mode 100644 cloud/package.json create mode 100644 cloud/packages/relay-contract/package.json create mode 100644 cloud/packages/relay-contract/src/admission-budgets.ts create mode 100644 cloud/packages/relay-contract/src/assignment-invariants.ts create mode 100644 cloud/packages/relay-contract/src/close-codes.ts create mode 100644 cloud/packages/relay-contract/src/contract.test.ts create mode 100644 cloud/packages/relay-contract/src/control-continuity.ts create mode 100644 cloud/packages/relay-contract/src/control-messages.ts create mode 100644 cloud/packages/relay-contract/src/credential-messages.ts create mode 100644 cloud/packages/relay-contract/src/director-messages.ts create mode 100644 cloud/packages/relay-contract/src/host-proof-transcript.ts create mode 100644 cloud/packages/relay-contract/src/index.ts create mode 100644 cloud/packages/relay-contract/src/persistence-invariants.test.ts create mode 100644 cloud/packages/relay-contract/src/persistence-invariants.ts create mode 100644 cloud/packages/relay-contract/src/protocol-limits.ts create mode 100644 cloud/packages/relay-contract/src/relay-regions.ts create mode 100644 cloud/packages/relay-contract/src/resume-confirmation-contract.ts create mode 100644 cloud/packages/relay-contract/src/splice-state-machine.ts create mode 100644 cloud/packages/relay-contract/src/wire-scalars.ts create mode 100644 cloud/packages/relay-contract/tsconfig.build.json create mode 100644 cloud/packages/relay-contract/tsconfig.json create mode 100644 cloud/pnpm-lock.yaml create mode 100644 cloud/pnpm-workspace.yaml create mode 100644 cloud/tsconfig.base.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b44287a9e18..dc1c1412470 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,3 +3,8 @@ /config/scripts/*localization*.mjs @brennanb2025 /config/scripts/*locale*.mjs @brennanb2025 /config/i18next.config.ts @brennanb2025 + +# The relay's deploy and operate surface: workspace, workflows, and the rollout lease action. +/cloud/ @Jinwoo-H +/.github/workflows/cloud-*.yml @Jinwoo-H +/.github/actions/cloud-sql-rollout-lease/ @Jinwoo-H diff --git a/.github/actions/cloud-sql-rollout-lease/README.md b/.github/actions/cloud-sql-rollout-lease/README.md new file mode 100644 index 00000000000..a06664b1f18 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/README.md @@ -0,0 +1,152 @@ +# Cloud SQL rollout lease + +A compare-and-swap lease on one Cloud Storage object, used to serialize Cloud SQL +**connection-budget** rollouts across two repositories. + +`concurrency.group: production-cloud-sql-rollout` only serializes runs inside a single repository. +Once the relay workflows live in `stablyai/orca` and the app workflows stay in +`stablyai/orca-cloud`, there are two independent queues pointed at one shared Cloud SQL instance. +`relay-cloud-sql-connection-budget.mjs` computes `rolloutOverlap` as a `Math.max` over the relay +director, api, auth and relay-cell candidates, which is only sound when exactly one rollout is in +flight. This lease is what keeps that assumption true. Keep the per-repo concurrency groups **and** +the lease; they solve different halves of the problem. + +## What it protects + +No workflow runs a Cloud SQL schema migration. Every locked workflow either deploys a Cloud Run +revision or applies a GCE instance template against the shared instance, so the lease must cover +**all rollouts**, not just migrations. + +## Usage + +The lease step must run **after** `google-github-actions/setup-gcloud`, and after +`actions/checkout` — `uses: ./.github/actions/...` resolves against the checked-out workspace. +It belongs in the first job of the workflow that holds a GCP credential, which is not always the +gate job: `deploy-relay-production-same-cap`'s gate runs no `gcloud`, so its first acquire happens +in the first cell job. + +```yaml +- uses: google-github-actions/auth@v2 + with: { workload_identity_provider: ..., service_account: ... } +- uses: google-github-actions/setup-gcloud@v2 +- uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock +``` + +Buckets and objects in use: + +| Environment | Bucket | Object | +| ----------- | -------------------------------------- | --------------------------------------------------- | +| production | `onorca-cloud-terraform-state` | `terraform/state/cloud-sql-rollout/production.lock` | +| staging | `onorca-cloud-staging-terraform-state` | `terraform/state/cloud-sql-rollout/staging.lock` | + +Workflows that serve both environments (`deploy-relay-asia-topology`, +`operate-relay-asia-admission`) select the pair with an `inputs.environment == 'production'` +ternary on both `bucket` and `object`. `deploy-staging` keeps its own `deploy-artifacts-staging` +concurrency group but takes the staging lease, because it rolls the staging API revision. + +The object sits beside `terraform/state/relay-fence-broker/.lock`. The IAM grant names both the +relay and app service accounts, so it is a **foundation-root** resource: `roles/storage.objectAdmin` +conditioned on the `terraform/state/cloud-sql-rollout/` prefix, **plus** an unconditioned +`roles/storage.legacyBucketReader`. Without the second role the generation-matched write fails in a +way that looks like a permissions flake. + +## One lease per run, not per job + +`deploy-relay-production-capacity` calls its reusable job six times and +`deploy-relay-production-same-cap` four times. Each call is a separate job on a separate runner, so +a naive per-job acquire/release would leave the object free between waves — for runs that have taken +up to 85 minutes. + +The lease is therefore keyed to the **run**, not the job. `holder-key` defaults to +`${{ github.repository }}/${{ github.run_id }}`, and a job that finds its own holder key on a live +lease **re-enters** it: the record is refreshed, not rejected. Every job in the chain acquires; only +the last one releases. + +```yaml +jobs: + gate: + steps: + - uses: ./.github/actions/cloud-sql-rollout-lease + with: { bucket: ..., object: ..., release: 'false' } # intermediate + + wave-1: # ... release: 'false' on every wave job + + release_lease: + needs: [gate, wave-1, wave-2, wave-3, wave-4] + if: always() + steps: + - uses: google-github-actions/auth@v2 + - uses: google-github-actions/setup-gcloud@v2 + - uses: ./.github/actions/cloud-sql-rollout-lease + with: { bucket: ..., object: ..., release: 'true' } # final +``` + +`release: 'false'` still acquires and still runs its `post` step; `post` only skips the delete. A +single-job workflow leaves `release` at its `true` default and needs no extra job. So does a +workflow whose several jobs can never hold the lease at once: `prove-relay-staging-capacity`'s two +lease-holding jobs are guarded by complementary `inputs.mode` conditions, and the contract test +checks that exclusivity rather than assuming it. + +If the final job never runs (runner killed, run cancelled hard), the lease expires on its TTL. + +## Timing + +- **TTL 35 minutes**, matching `apps/relay-fence-broker/src/mutation-lease.ts`. +- **Renewal every 5 minutes.** `main` spawns a detached background Node process that rewrites + `expires_at` on the same generation-matched path; `post` kills it by pid read back from + `$GITHUB_STATE`. The renewer stops on its own the moment the object stops being ours, and has a + six-hour backstop in case `post` never runs. Its log is written to + `$RUNNER_TEMP/cloud-sql-rollout-lease-renewer.log` and echoed by `post`. +- Renewal is mandatory, not optional: capacity runs have taken 85 minutes, well past any sane TTL. + +## Failure behaviour + +| Situation | Behaviour | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| Object absent | Acquire with `ifGenerationMatch: 0`. | +| Live lease, our own holder key | Re-enter. Refresh `expires_at`, keep `acquired_at`. Never fails. | +| Live lease, another holder | **Fail the job immediately**, printing the holder's repository, workflow and run URL. Never queues, never steals. | +| Expired lease | Take over with the observed generation and emit `::warning::` naming the stale holder. | +| `412` on write | Someone raced us. Fail as a conflict. | +| Bucket unreachable, `403`, `5xx` | **Fail closed.** | +| Record present but unparseable | **Fail closed.** A record we cannot read is never treated as free; an operator must inspect and delete it. | +| Release finds a foreign holder | Warn and leave it alone. Our lease had already expired. | +| Release fails | Warn only. `post` never fails a job over a release; the TTL bounds the damage. | + +## Why `monitor-relay-production` must not use this + +`monitor-relay-production` is in the `production-cloud-sql-rollout` concurrency group but is +**read-only**: its identity holds only monitoring, logging, Cloud SQL and compute _viewer_ roles, +and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget. +Putting it on the durable lease would let a monitoring run block a real rollout, and a rollout block +monitoring exactly when an operator most needs it. Keep its same-repo concurrency group; keep it off +the lease. The lock census contract test records it in the not-a-candidate map with this reason. + +## Token acquisition + +`gcloud auth print-access-token`, not a hand-rolled exchange of the `external_account` credentials +file. Every consuming workflow already runs `setup-gcloud`, gcloud already handles every ADC flavour +including the service-account impersonation leg, and this action must stay zero-dependency because +it is duplicated by hand into the public repo. The GCE metadata server that +`apps/relay-fence-broker/src/google-metadata.ts` uses does **not** exist on GitHub or Blacksmith +runners; only the compare-and-swap algorithm is shared with the fence broker. + +## Duplication + +This directory is copied verbatim into `stablyai/orca`. It has no `package.json`, no +`node_modules`, and imports nothing outside itself — `action-contract.test.mjs` enforces all three. +Cross-repo consumption via `uses: stablyai/orca/.github/actions/...@` was rejected: it would +put public-repo code inside private app deploys that hold a production credential, and neither +repository protects `main` today. + +## Tests + +``` +node --test .github/actions/cloud-sql-rollout-lease/ +``` + +`storage-lease.test.mjs` drives the real compare-and-swap path against an in-memory Cloud Storage +fake that enforces generations. No network. diff --git a/.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs b/.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs new file mode 100644 index 00000000000..631daef7d3e --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict' +import { readFileSync, readdirSync } from 'node:fs' +import { test } from 'node:test' + +const here = new URL('./', import.meta.url) +const action = readFileSync(new URL('action.yml', here), 'utf8') +const modules = readdirSync(here).filter((name) => name.endsWith('.mjs')) +const shipped = modules.filter((name) => !name.endsWith('.test.mjs')) + +test('is a node24 JavaScript action with an always-run post step', () => { + // A composite action has no `post:`, so the lease could never be released on cancel or failure. + assert.match(action, /^ {2}using: node24$/m) + assert.doesNotMatch(action, /using: composite/) + assert.match(action, /^ {2}main: main\.mjs$/m) + assert.match(action, /^ {2}post: post\.mjs$/m) + assert.match(action, /^ {2}post-if: always\(\)$/m) +}) + +test('declares the inputs the wave-chain callers depend on', () => { + for (const input of ['bucket:', 'object:', 'holder-key:', 'release:']) { + assert.match(action, new RegExp(`^ {2}${input}$`, 'm'), input) + } + assert.match(action, /default: \$\{\{ github\.repository \}\}\/\$\{\{ github\.run_id \}\}/) + assert.match(action, /default: 'true'/) +}) + +test('stays self-contained so it can be duplicated into the public repo', () => { + assert.deepEqual( + readdirSync(here).filter((name) => name === 'package.json' || name === 'node_modules'), + [], + 'the action must run with zero installed dependencies' + ) + for (const name of modules) { + const source = readFileSync(new URL(name, here), 'utf8') + for (const match of source.matchAll(/^import\b[\s\S]*?from '([^']+)'/gm)) { + const specifier = match[1] + const local = specifier.startsWith('.') + assert.ok( + specifier.startsWith('node:') || (local && !specifier.includes('..')), + `${name} imports ${specifier}; only node: builtins and same-directory modules are allowed` + ) + } + } +}) + +test('never reaches for the GCE metadata server', () => { + // Runners have no metadata.google.internal; the fence broker's token path must not be copied. + for (const name of shipped) { + const source = readFileSync(new URL(name, here), 'utf8') + assert.doesNotMatch(source, /metadata\.google\.internal/, name) + } +}) diff --git a/.github/actions/cloud-sql-rollout-lease/action.yml b/.github/actions/cloud-sql-rollout-lease/action.yml new file mode 100644 index 00000000000..afd4b8efee0 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/action.yml @@ -0,0 +1,41 @@ +name: Cloud SQL rollout lease +description: >- + Serialize Cloud SQL connection-budget rollouts across repositories with a compare-and-swap lease + on a Cloud Storage object. Fails immediately when another run holds the lease; never queues, + never steals. + +inputs: + bucket: + description: Terraform state bucket that holds the lease object. + required: true + object: + description: Lease object name, e.g. terraform/state/cloud-sql-rollout/production.lock + required: true + holder-key: + description: >- + Identity that owns the lease. Every job in one run must pass the same value; a job that finds + its own holder key on a live lease re-enters it instead of failing. + required: false + default: ${{ github.repository }}/${{ github.run_id }} + release: + description: >- + Release the lease in the post step. Set to "false" on every job of a multi-job wave except the + final always() job, which sets "true". + required: false + default: 'true' + +outputs: + holder-key: + description: The holder key written to the lease object. + generation: + description: Cloud Storage generation of the lease object after acquisition. + expires-at: + description: ISO-8601 instant at which the lease expires without renewal. + reentrant: + description: '"true" when this job re-entered a lease its own run already held.' + +runs: + using: node24 + main: main.mjs + post: post.mjs + post-if: always() diff --git a/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs b/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs new file mode 100644 index 00000000000..8ff2bc58646 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs @@ -0,0 +1,46 @@ +import { execFileSync } from 'node:child_process' + +// DESIGN CHOICE: shell out to `gcloud auth print-access-token` instead of exchanging the +// external_account credentials file that google-github-actions/auth writes. +// +// Every workflow on the Cloud SQL rollout lease already runs google-github-actions/setup-gcloud +// right after auth (verified across all 11 mutating members), so gcloud is on PATH and already +// bound to the federated identity. Doing the exchange ourselves would mean reimplementing the STS +// token swap plus the service-account impersonation leg, in an action that must stay +// zero-dependency and is duplicated by hand into a second repo. gcloud already handles every ADC +// flavour and refreshes on its own. The metadata server is not an option: it does not exist on +// GitHub or Blacksmith runners. +// +// Consequence, documented in the README: the lease step MUST come after setup-gcloud. + +const TOKEN_REUSE_MS = 40 * 60 * 1_000 // GCP access tokens live ~60 min; re-mint well before that. + +export function createAccessTokenSource({ run = runGcloud, now = Date.now } = {}) { + let cached = null + return () => { + if (cached && cached.mintedAt + TOKEN_REUSE_MS > now()) { + return cached.token + } + const token = run() + if (!token) { + throw new Error('gcloud auth print-access-token returned an empty token') + } + cached = { token, mintedAt: now() } + return token + } +} + +function runGcloud() { + const binary = process.platform === 'win32' ? 'gcloud.cmd' : 'gcloud' + try { + return execFileSync(binary, ['auth', 'print-access-token'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 60_000 + }).trim() + } catch (error) { + // Never surface stdout; it is the token on success and noise on failure. + const detail = String(error?.stderr ?? '').trim() || error?.message || 'unknown failure' + throw new Error(`could not mint a GCP access token via gcloud: ${detail}`) + } +} diff --git a/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs b/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs new file mode 100644 index 00000000000..105d0b3bff0 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs @@ -0,0 +1,20 @@ +// Who we claim to be on the lease object. Shared by main and the detached renewer so both agree +// on the holder key without re-deriving it from a different set of environment variables. + +export function holderIdentity(explicitHolderKey) { + const repository = process.env.GITHUB_REPOSITORY ?? 'unknown' + const runId = process.env.GITHUB_RUN_ID ?? 'unknown' + const server = process.env.GITHUB_SERVER_URL ?? 'https://github.com' + const holderKey = explicitHolderKey || `${repository}/${runId}` + if (/[\r\n]/.test(holderKey)) { + throw new Error('holder-key must be single-line') + } + return { + holderKey, + repository, + workflow: process.env.GITHUB_WORKFLOW ?? 'unknown', + runId, + runUrl: `${server}/${repository}/actions/runs/${runId}`, + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? 'unknown' + } +} diff --git a/.github/actions/cloud-sql-rollout-lease/main.mjs b/.github/actions/cloud-sql-rollout-lease/main.mjs new file mode 100644 index 00000000000..0a429763a73 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/main.mjs @@ -0,0 +1,81 @@ +import { spawn } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { createAccessTokenSource } from './gcloud-access-token.mjs' +import { holderIdentity } from './holder-identity.mjs' +import { fail, input, notice, renewerLogPath, saveState, setOutput, warn } from './runner-state.mjs' +import { CloudSqlRolloutLease, LeaseConflict, describeHolder } from './storage-lease.mjs' + +const bucket = input('bucket') +const objectName = input('object') +const release = input('release') !== 'false' + +if (!bucket || !objectName) { + fail('cloud-sql-rollout-lease requires both `bucket` and `object`') + process.exit(1) +} + +const holder = holderIdentity(input('holder-key')) +const lease = new CloudSqlRolloutLease({ + bucket, + objectName, + accessToken: createAccessTokenSource() +}) + +let claim +try { + claim = await lease.acquire(holder) +} catch (error) { + if (error instanceof LeaseConflict) { + fail( + `${error.message}. Cloud SQL rollouts are serialized across repositories; this run will not queue or steal the lease. Wait for the holder to finish, then re-run.` + ) + if (error.holder) { + console.log(`Lease holder repository: ${error.holder.repository}`) + console.log(`Lease holder workflow: ${error.holder.workflow}`) + console.log(`Lease holder run: ${error.holder.run_url}`) + } + } else { + // Bucket unreachable, permission denied, unreadable record: fail closed. + fail(`could not acquire ${lease.uri}: ${error.message}`) + } + process.exit(1) +} + +// Persist before anything else can throw, so `post` always releases what we hold. +saveState('acquired', 'true') +saveState('bucket', bucket) +saveState('object', objectName) +saveState('holder_key', holder.holderKey) +saveState('release', release ? 'true' : 'false') + +setOutput('holder-key', holder.holderKey) +setOutput('generation', claim.generation) +setOutput('expires-at', new Date(claim.record.expires_at).toISOString()) +setOutput('reentrant', claim.state === 'reentrant' ? 'true' : 'false') + +if (claim.state === 'reentrant') { + notice( + `Re-entered the Cloud SQL rollout lease on ${lease.uri} already held by this run; refreshed to ${new Date(claim.record.expires_at).toISOString()}.` + ) +} else if (claim.state === 'takeover') { + notice(`Took over ${lease.uri} from ${describeHolder(claim.previous)}.`) +} else { + notice( + `Acquired ${lease.uri} until ${new Date(claim.record.expires_at).toISOString()} (holder ${holder.holderKey}).` + ) +} + +try { + const renewer = spawn( + process.execPath, + [fileURLToPath(new URL('./renew.mjs', import.meta.url)), bucket, objectName, holder.holderKey], + { detached: true, stdio: 'ignore', env: process.env } + ) + renewer.unref() + saveState('renewer_pid', String(renewer.pid)) + const log = renewerLogPath() + notice(`Lease renewer running as pid ${renewer.pid}${log ? `, logging to ${log}` : ''}.`) +} catch (error) { + // A missing renewer is survivable for short jobs; the TTL still covers 35 minutes. + warn(`could not start the lease renewer: ${error.message}. The lease will expire on its TTL.`) +} diff --git a/.github/actions/cloud-sql-rollout-lease/post.mjs b/.github/actions/cloud-sql-rollout-lease/post.mjs new file mode 100644 index 00000000000..5be970a34b3 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/post.mjs @@ -0,0 +1,81 @@ +import { readFileSync } from 'node:fs' +import { createAccessTokenSource } from './gcloud-access-token.mjs' +import { notice, renewerLogPath, savedState, warn } from './runner-state.mjs' +import { CloudSqlRolloutLease, describeHolder } from './storage-lease.mjs' + +stopRenewer() +printRenewerLog() + +if (savedState('acquired') !== 'true') { + notice('No Cloud SQL rollout lease was acquired by this step; nothing to release.') + process.exit(0) +} + +const bucket = savedState('bucket') +const objectName = savedState('object') +const holderKey = savedState('holder_key') + +if (savedState('release') !== 'true') { + notice( + `Holding gs://${bucket}/${objectName} for the rest of run ${holderKey}; a later job with release=true must free it.` + ) + process.exit(0) +} + +const lease = new CloudSqlRolloutLease({ + bucket, + objectName, + accessToken: createAccessTokenSource() +}) + +try { + const result = await lease.release(holderKey) + if (result.released) { + notice(`Released ${lease.uri} at generation ${result.generation}.`) + } else if (result.reason === 'absent') { + notice(`${lease.uri} was already gone; nothing to release.`) + } else if (result.reason === 'foreign') { + warn( + `${lease.uri} is now held by ${describeHolder(result.holder)}; leaving it alone. Our lease had already expired.` + ) + } else { + warn(`${lease.uri} changed while releasing it; leaving it to expire on its TTL.`) + } +} catch (error) { + // Never fail a job in post over a release; the TTL bounds the damage to 35 minutes. + warn(`could not release ${lease.uri}: ${error.message}. It will expire on its TTL.`) +} + +function stopRenewer() { + const pid = Number(savedState('renewer_pid')) + if (!Number.isInteger(pid) || pid <= 0) { + return + } + try { + process.kill(pid, 'SIGTERM') + notice(`Stopped the lease renewer (pid ${pid}).`) + } catch (error) { + if (error?.code !== 'ESRCH') { + warn(`could not stop the lease renewer ${pid}: ${error.message}`) + } + } +} + +function printRenewerLog() { + const path = renewerLogPath() + if (!path) { + return + } + let text = '' + try { + text = readFileSync(path, 'utf8') + } catch { + return + } + if (!text.trim()) { + return + } + console.log('::group::Cloud SQL rollout lease renewer log') + console.log(text.trimEnd()) + console.log('::endgroup::') +} diff --git a/.github/actions/cloud-sql-rollout-lease/renew.mjs b/.github/actions/cloud-sql-rollout-lease/renew.mjs new file mode 100644 index 00000000000..cc8e543ba4f --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/renew.mjs @@ -0,0 +1,73 @@ +import { appendFileSync } from 'node:fs' +import { createAccessTokenSource } from './gcloud-access-token.mjs' +import { renewerLogPath } from './runner-state.mjs' +import { CloudSqlRolloutLease, RENEW_INTERVAL_MS } from './storage-lease.mjs' + +// Detached renewer. `main` spawns it, `post` kills it. It rewrites expires_at on the same +// generation-matched path as acquisition, and stops the moment the object stops being ours. + +const MAX_LIFETIME_MS = 6 * 60 * 60 * 1_000 // Backstop if post never runs (runner killed). + +const [bucket, objectName, holderKey] = process.argv.slice(2) +const logPath = renewerLogPath() +const startedAt = Date.now() + +function log(message) { + if (!logPath) { + return + } + try { + appendFileSync(logPath, `${new Date().toISOString()} ${message}\n`) + } catch { + // A renewer that cannot log must still renew. + } +} + +if (!bucket || !objectName || !holderKey) { + log('renewer started without bucket/object/holder-key; exiting') + process.exit(1) +} + +const lease = new CloudSqlRolloutLease({ + bucket, + objectName, + accessToken: createAccessTokenSource(), + warn: (message) => log(`warning ${message}`) +}) + +let stopping = false +for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) { + process.on(signal, () => { + stopping = true + log(`received ${signal}; stopping`) + process.exit(0) + }) +} + +log(`renewer started for ${lease.uri} holder=${holderKey} interval=${RENEW_INTERVAL_MS}ms`) + +while (!stopping) { + await new Promise((resolve) => { + setTimeout(resolve, RENEW_INTERVAL_MS) + }) + if (stopping) { + break + } + if (Date.now() - startedAt > MAX_LIFETIME_MS) { + log('renewer hit its maximum lifetime; stopping so the lease can expire') + break + } + try { + const result = await lease.renew(holderKey) + if (!result.renewed) { + log(`lease is no longer ours (${result.reason}); stopping`) + break + } + log( + `renewed until ${new Date(result.record.expires_at).toISOString()} at generation ${result.generation}` + ) + } catch (error) { + // Transient GCS or token failures are retried on the next tick; the TTL covers 7 misses. + log(`renewal attempt failed: ${error.message}`) + } +} diff --git a/.github/actions/cloud-sql-rollout-lease/runner-state.mjs b/.github/actions/cloud-sql-rollout-lease/runner-state.mjs new file mode 100644 index 00000000000..ec36a16e0f2 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/runner-state.mjs @@ -0,0 +1,55 @@ +import { appendFileSync } from 'node:fs' + +// Action-state and output plumbing via the runner's file protocol, so the action needs no +// @actions/core dependency. Values are single-line by construction; anything else is rejected. + +/** The detached renewer's stdio is ignored, so it appends here instead and `post` echoes it. */ +export function renewerLogPath() { + const dir = process.env.RUNNER_TEMP + return dir ? `${dir}/cloud-sql-rollout-lease-renewer.log` : null +} + +export function input(name) { + return (process.env[`INPUT_${name.replace(/ /g, '_').toUpperCase()}`] ?? '').trim() +} + +export function savedState(name) { + return (process.env[`STATE_${name}`] ?? '').trim() +} + +export function saveState(name, value) { + appendToEnvFile('GITHUB_STATE', name, value) +} + +export function setOutput(name, value) { + appendToEnvFile('GITHUB_OUTPUT', name, value) +} + +export function notice(message) { + console.log(`::notice::${oneLine(message)}`) +} + +export function warn(message) { + console.log(`::warning::${oneLine(message)}`) +} + +export function fail(message) { + console.log(`::error::${oneLine(message)}`) + process.exitCode = 1 +} + +function appendToEnvFile(variable, name, value) { + const text = String(value) + if (/[\r\n]/.test(text)) { + throw new Error(`${name} must be single-line`) + } + const path = process.env[variable] + if (!path) { + return + } // Running outside a runner (local smoke run); nothing to persist. + appendFileSync(path, `${name}=${text}\n`) +} + +function oneLine(message) { + return String(message).replace(/\r?\n/g, ' ') +} diff --git a/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs b/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs new file mode 100644 index 00000000000..44312377ee3 --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs @@ -0,0 +1,240 @@ +// Compare-and-swap lease over a single Cloud Storage object. +// +// Ported from apps/relay-fence-broker/src/mutation-lease.ts rather than imported: this action is +// duplicated verbatim into stablyai/orca, so it must carry no repo-local imports. Only the +// algorithm is shared (read metadata -> write with ifGenerationMatch -> 412 is a conflict -> +// generation-matched delete -> an expired record is free). The broker's token path is NOT shared; +// it reads the GCE metadata server, which does not exist on Actions runners. + +export const LEASE_TTL_MS = 35 * 60 * 1_000 +export const RENEW_INTERVAL_MS = 5 * 60 * 1_000 + +const GENERATION = /^[1-9][0-9]{0,30}$/ + +export class LeaseConflict extends Error { + constructor(message, holder) { + super(message) + this.name = 'LeaseConflict' + this.holder = holder ?? null + } +} + +/** A live record we cannot parse is never treated as free; wedging beats double-rollout. */ +export class LeaseUnreadable extends Error { + constructor(message) { + super(message) + this.name = 'LeaseUnreadable' + } +} + +function parseRecord(raw) { + if (!raw || typeof raw !== 'object') { + return null + } + if (typeof raw.holder_key !== 'string' || raw.holder_key.length === 0) { + return null + } + if (!Number.isSafeInteger(raw.acquired_at) || !Number.isSafeInteger(raw.expires_at)) { + return null + } + return { + repository: typeof raw.repository === 'string' ? raw.repository : 'unknown', + workflow: typeof raw.workflow === 'string' ? raw.workflow : 'unknown', + run_id: typeof raw.run_id === 'string' ? raw.run_id : 'unknown', + run_url: typeof raw.run_url === 'string' ? raw.run_url : 'unknown', + run_attempt: typeof raw.run_attempt === 'string' ? raw.run_attempt : 'unknown', + acquired_at: raw.acquired_at, + expires_at: raw.expires_at, + holder_key: raw.holder_key + } +} + +function describe(record) { + return `${record.repository} / ${record.workflow} (run ${record.run_id}, attempt ${record.run_attempt}) ${record.run_url}` +} + +export class CloudSqlRolloutLease { + #bucket + #objectName + #accessToken + #fetcher + #now + #warn + + constructor({ + bucket, + objectName, + accessToken, + fetcher = fetch, + now = Date.now, + warn = (message) => console.log(`::warning::${message}`) + }) { + this.#bucket = bucket + this.#objectName = objectName + this.#accessToken = accessToken + this.#fetcher = fetcher + this.#now = now + this.#warn = warn + } + + get uri() { + return `gs://${this.#bucket}/${this.#objectName}` + } + + async acquire(holder) { + const existing = await this.read() + const now = this.#now() + if (!existing) { + return this.#claim(holder, '0', now, now, 'created') + } + if (existing.record.holder_key === holder.holderKey) { + // Same run, another job in the wave chain. Refresh, never fail. + return this.#claim( + holder, + existing.generation, + existing.record.acquired_at, + now, + 'reentrant', + existing.record + ) + } + if (existing.record.expires_at > now) { + throw new LeaseConflict( + `${this.uri} is held by ${describe(existing.record)} until ${new Date(existing.record.expires_at).toISOString()}`, + existing.record + ) + } + this.#warn( + `Taking over an expired Cloud SQL rollout lease on ${this.uri}. Stale holder: ${describe(existing.record)}, expired ${new Date(existing.record.expires_at).toISOString()}.` + ) + return this.#claim(holder, existing.generation, now, now, 'takeover', existing.record) + } + + async renew(holderKey) { + const existing = await this.read() + if (!existing) { + return { renewed: false, reason: 'absent' } + } + if (existing.record.holder_key !== holderKey) { + return { renewed: false, reason: 'foreign' } + } + const now = this.#now() + const record = { ...existing.record, expires_at: now + LEASE_TTL_MS } + const written = await this.#write(record, existing.generation) + return { renewed: true, generation: written.generation, record } + } + + async release(holderKey) { + const existing = await this.read() + if (!existing) { + return { released: false, reason: 'absent' } + } + if (existing.record.holder_key !== holderKey) { + return { released: false, reason: 'foreign', holder: existing.record } + } + const response = await this.#fetcher( + `${this.#metadataUrl()}?ifGenerationMatch=${encodeURIComponent(existing.generation)}`, + { method: 'DELETE', headers: { Authorization: `Bearer ${await this.#token()}` } } + ) + if (response.status === 412) { + return { released: false, reason: 'conflict' } + } + if (!response.ok && response.status !== 404) { + throw new Error(`lease release failed: ${response.status}`) + } + return { released: true, generation: existing.generation } + } + + async read() { + const token = await this.#token() + const metadataResponse = await this.#fetcher(this.#metadataUrl(), { + headers: { Authorization: `Bearer ${token}` } + }) + if (metadataResponse.status === 404) { + return null + } + if (!metadataResponse.ok) { + throw new Error(`lease inspection failed: ${metadataResponse.status}`) + } + const metadata = await metadataResponse.json() + if (!GENERATION.test(metadata?.generation ?? '')) { + throw new LeaseUnreadable(`${this.uri} has no valid generation`) + } + const bodyResponse = await this.#fetcher(`${this.#metadataUrl()}?alt=media`, { + headers: { Authorization: `Bearer ${token}` } + }) + if (bodyResponse.status === 404) { + return null + } + if (!bodyResponse.ok) { + throw new Error(`lease body read failed: ${bodyResponse.status}`) + } + let raw = null + try { + raw = await bodyResponse.json() + } catch { + raw = null + } + const record = parseRecord(raw) + if (!record) { + throw new LeaseUnreadable( + `${this.uri} holds an unreadable lease record; an operator must inspect and delete it before rollouts can resume` + ) + } + return { generation: metadata.generation, record } + } + + async #claim(holder, generation, acquiredAt, now, state, previous) { + const record = { + repository: holder.repository, + workflow: holder.workflow, + run_id: holder.runId, + run_url: holder.runUrl, + run_attempt: holder.runAttempt, + acquired_at: acquiredAt, + expires_at: now + LEASE_TTL_MS, + holder_key: holder.holderKey + } + const written = await this.#write(record, generation) + return { state, generation: written.generation, record, previous: previous ?? null } + } + + async #write(record, generation) { + const response = await this.#fetcher( + `${this.#uploadUrl()}&ifGenerationMatch=${encodeURIComponent(generation)}`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${await this.#token()}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify(record) + } + ) + if (response.status === 412) { + throw new LeaseConflict(`${this.uri} changed concurrently while we were claiming it`) + } + if (!response.ok) { + throw new Error(`lease write failed: ${response.status}`) + } + const metadata = await response.json() + if (!GENERATION.test(metadata?.generation ?? '')) { + throw new LeaseUnreadable(`${this.uri} write returned no valid generation`) + } + return { generation: metadata.generation } + } + + async #token() { + return typeof this.#accessToken === 'function' ? await this.#accessToken() : this.#accessToken + } + + #metadataUrl() { + return `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(this.#bucket)}/o/${encodeURIComponent(this.#objectName)}` + } + + #uploadUrl() { + return `https://storage.googleapis.com/upload/storage/v1/b/${encodeURIComponent(this.#bucket)}/o?uploadType=media&name=${encodeURIComponent(this.#objectName)}` + } +} + +export const describeHolder = describe diff --git a/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs b/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs new file mode 100644 index 00000000000..f13e01517be --- /dev/null +++ b/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs @@ -0,0 +1,324 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { createAccessTokenSource } from './gcloud-access-token.mjs' +import { + CloudSqlRolloutLease, + LEASE_TTL_MS, + LeaseConflict, + LeaseUnreadable +} from './storage-lease.mjs' + +const BUCKET = 'onorca-cloud-terraform-state' +const OBJECT = 'terraform/state/cloud-sql-rollout/production.lock' +const NOW = 1_756_000_000_000 + +/** + * Enough of the Cloud Storage JSON API to exercise real compare-and-swap semantics: generations + * increment, ifGenerationMatch is enforced, and a mismatch is a 412. `faults` injects failures. + */ +function fakeStorage({ object = null, faults = [] } = {}) { + const state = { object, requests: [] } + const fetcher = async (rawUrl, init = {}) => { + const url = new URL(rawUrl) + const method = init.method ?? 'GET' + const record = { method, url, path: url.pathname, search: url.searchParams } + state.requests.push(record) + const fault = faults.find((candidate) => candidate.when(record)) + if (fault) { + return json(fault.status, fault.body ?? {}) + } + + if (url.pathname.startsWith('/upload/')) { + const want = url.searchParams.get('ifGenerationMatch') + const have = state.object ? state.object.generation : '0' + if (want !== have) { + return json(412, {}) + } + const generation = String(Number(have === '0' ? '1000' : have) + 1) + state.object = { generation, body: JSON.parse(init.body) } + return json(200, { generation }) + } + if (method === 'DELETE') { + if (!state.object) { + return json(404, {}) + } + if (url.searchParams.get('ifGenerationMatch') !== state.object.generation) { + return json(412, {}) + } + state.object = null + return json(204, {}) + } + if (!state.object) { + return json(404, {}) + } + if (url.searchParams.get('alt') === 'media') { + return json(200, state.object.body) + } + return json(200, { generation: state.object.generation }) + } + return { state, fetcher } +} + +function json(status, body) { + return { + status, + ok: status >= 200 && status < 300, + json: async () => body + } +} + +function storedRecord({ + holderKey, + expiresAt, + repository = 'stablyai/orca', + workflow = 'Deploy Relay Production' +}) { + return { + repository, + workflow, + run_id: '9001', + run_url: 'https://github.com/stablyai/orca/actions/runs/9001', + run_attempt: '1', + acquired_at: NOW - 60_000, + expires_at: expiresAt, + holder_key: holderKey + } +} + +function leaseFor(storage, { warn = () => {} } = {}) { + return new CloudSqlRolloutLease({ + bucket: BUCKET, + objectName: OBJECT, + accessToken: 'test-token', + fetcher: storage.fetcher, + now: () => NOW, + warn + }) +} + +const HOLDER = { + holderKey: 'stablyai/orca-cloud/42', + repository: 'stablyai/orca-cloud', + workflow: 'Deploy Relay Production Same-Cap', + runId: '42', + runUrl: 'https://github.com/stablyai/orca-cloud/actions/runs/42', + runAttempt: '1' +} + +test('acquires a lease on an empty object with ifGenerationMatch=0', async () => { + const storage = fakeStorage() + const claim = await leaseFor(storage).acquire(HOLDER) + + assert.equal(claim.state, 'created') + const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/')) + assert.equal(upload.search.get('ifGenerationMatch'), '0') + assert.equal(upload.search.get('name'), OBJECT) + assert.deepEqual(storage.state.object.body, { + repository: 'stablyai/orca-cloud', + workflow: 'Deploy Relay Production Same-Cap', + run_id: '42', + run_url: 'https://github.com/stablyai/orca-cloud/actions/runs/42', + run_attempt: '1', + acquired_at: NOW, + expires_at: NOW + LEASE_TTL_MS, + holder_key: 'stablyai/orca-cloud/42' + }) +}) + +test('refuses a live lease held by another run and never writes', async () => { + const storage = fakeStorage({ + object: { + generation: '1500', + body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 }) + } + }) + + const error = await leaseFor(storage) + .acquire(HOLDER) + .catch((thrown) => thrown) + + assert.ok(error instanceof LeaseConflict) + assert.equal(error.holder.repository, 'stablyai/orca') + assert.equal(error.holder.run_url, 'https://github.com/stablyai/orca/actions/runs/9001') + assert.equal( + storage.state.requests.filter((request) => request.method !== 'GET').length, + 0, + 'a foreign live lease must not be written' + ) + assert.equal(storage.state.object.generation, '1500') +}) + +test('re-enters a live lease this run already holds and extends it', async () => { + const storage = fakeStorage({ + object: { + generation: '1500', + body: storedRecord({ holderKey: HOLDER.holderKey, expiresAt: NOW + 60_000 }) + } + }) + const warnings = [] + const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire( + HOLDER + ) + + assert.equal(claim.state, 'reentrant') + assert.deepEqual(warnings, [], 're-entering our own lease is not a takeover') + assert.equal(claim.record.acquired_at, NOW - 60_000, 'original acquisition time is preserved') + assert.equal(claim.record.expires_at, NOW + LEASE_TTL_MS) + const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/')) + assert.equal(upload.search.get('ifGenerationMatch'), '1500') + assert.equal(storage.state.object.generation, '1501') +}) + +test('takes over an expired lease and warns naming the stale holder', async () => { + const storage = fakeStorage({ + object: { + generation: '1500', + body: storedRecord({ + holderKey: 'stablyai/orca/9001', + expiresAt: NOW - 1, + repository: 'stablyai/orca', + workflow: 'Deploy Relay Production Capacity' + }) + } + }) + const warnings = [] + const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire( + HOLDER + ) + + assert.equal(claim.state, 'takeover') + assert.equal(warnings.length, 1) + assert.match(warnings[0], /stablyai\/orca/) + assert.match(warnings[0], /Deploy Relay Production Capacity/) + assert.match(warnings[0], /actions\/runs\/9001/) + const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/')) + assert.equal(upload.search.get('ifGenerationMatch'), '1500') + assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey) +}) + +test('releases with a generation match and leaves the object gone', async () => { + const storage = fakeStorage() + const lease = leaseFor(storage) + const claim = await lease.acquire(HOLDER) + + const released = await lease.release(HOLDER.holderKey) + + assert.deepEqual(released, { released: true, generation: claim.generation }) + const remove = storage.state.requests.find((request) => request.method === 'DELETE') + assert.equal(remove.search.get('ifGenerationMatch'), claim.generation) + assert.equal(storage.state.object, null) +}) + +test('refuses to release a lease another run now holds', async () => { + const storage = fakeStorage({ + object: { + generation: '1500', + body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 }) + } + }) + + const released = await leaseFor(storage).release(HOLDER.holderKey) + + assert.equal(released.released, false) + assert.equal(released.reason, 'foreign') + assert.equal(storage.state.object.generation, '1500') +}) + +test('fails closed when the bucket answers 5xx', async () => { + const storage = fakeStorage({ + faults: [{ when: (request) => request.method === 'GET', status: 503 }] + }) + + const error = await leaseFor(storage) + .acquire(HOLDER) + .catch((thrown) => thrown) + + assert.match(error.message, /lease inspection failed: 503/) + assert.equal(storage.state.requests.filter((request) => request.method !== 'GET').length, 0) +}) + +test('fails closed when permission is denied', async () => { + const storage = fakeStorage({ + faults: [{ when: (request) => request.method === 'GET', status: 403 }] + }) + + const error = await leaseFor(storage) + .acquire(HOLDER) + .catch((thrown) => thrown) + + assert.match(error.message, /lease inspection failed: 403/) +}) + +test('treats an unreadable record as held, not free', async () => { + const storage = fakeStorage({ + object: { generation: '1500', body: { holder_key: 'stablyai/orca/9001' } } + }) + + const error = await leaseFor(storage) + .acquire(HOLDER) + .catch((thrown) => thrown) + + assert.ok(error instanceof LeaseUnreadable) + assert.equal(storage.state.object.generation, '1500') +}) + +test('reports a 412 during acquisition as a conflict', async () => { + const storage = fakeStorage({ + faults: [{ when: (request) => request.path.startsWith('/upload/'), status: 412 }] + }) + + const error = await leaseFor(storage) + .acquire(HOLDER) + .catch((thrown) => thrown) + + assert.ok(error instanceof LeaseConflict) + assert.match(error.message, /changed concurrently/) +}) + +test('renewal rewrites only expires_at on the observed generation', async () => { + const storage = fakeStorage() + const lease = leaseFor(storage) + await lease.acquire(HOLDER) + storage.state.object.body.expires_at = NOW - 1 + + const renewed = await lease.renew(HOLDER.holderKey) + + assert.equal(renewed.renewed, true) + assert.equal(storage.state.object.body.expires_at, NOW + LEASE_TTL_MS) + assert.equal(storage.state.object.body.acquired_at, NOW) + assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey) +}) + +test('renewal stops once the object belongs to someone else', async () => { + const storage = fakeStorage({ + object: { + generation: '1500', + body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 }) + } + }) + + assert.deepEqual(await leaseFor(storage).renew(HOLDER.holderKey), { + renewed: false, + reason: 'foreign' + }) +}) + +test('the access token source re-mints only after the reuse window', () => { + let clock = 0 + let mints = 0 + const source = createAccessTokenSource({ + run: () => `token-${++mints}`, + now: () => clock + }) + + assert.equal(source(), 'token-1') + clock = 39 * 60 * 1_000 + assert.equal(source(), 'token-1') + clock = 41 * 60 * 1_000 + assert.equal(source(), 'token-2') +}) + +test('the access token source rejects an empty gcloud response', () => { + const source = createAccessTokenSource({ run: () => '' }) + assert.throws(() => source(), /empty token/) +}) diff --git a/.github/scripts/check-root-directory-entries.mjs b/.github/scripts/check-root-directory-entries.mjs index 5e7dcf3f0ff..5b63326691d 100644 --- a/.github/scripts/check-root-directory-entries.mjs +++ b/.github/scripts/check-root-directory-entries.mjs @@ -13,6 +13,10 @@ function readRootEntries(sha) { return stdout.split('\0').filter(Boolean) } +// Why: the Cloud workspace import is the one reviewed root addition; it stays +// listed until it lands on main, after which the base tree carries it. +const REVIEWED_ROOT_ENTRIES = new Set(['cloud']) + function checkRootDirectoryEntries(argv) { if (argv.length !== 2) { console.error(`Usage: ${process.argv[1]} `) @@ -21,7 +25,9 @@ function checkRootDirectoryEntries(argv) { const [baseSha, headSha] = argv const baseEntries = new Set(readRootEntries(baseSha)) - const blockedEntries = readRootEntries(headSha).filter((entry) => !baseEntries.has(entry)) + const blockedEntries = readRootEntries(headSha).filter( + (entry) => !baseEntries.has(entry) && !REVIEWED_ROOT_ENTRIES.has(entry) + ) if (blockedEntries.length === 0) { console.log('Root directory guard passed: no new root-level files or folders.') diff --git a/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml b/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml new file mode 100644 index 00000000000..29e68510b21 --- /dev/null +++ b/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml @@ -0,0 +1,676 @@ +name: Bootstrap Relay Staging Capacity + +on: + workflow_dispatch: + inputs: + confirmation: + description: Enter BOOTSTRAP_STAGING_CAPACITY + required: true + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: relay-staging-mutation + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + bootstrap: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: staging + env: + GCP_PROJECT_ID: onorca-cloud-staging + GCP_REGION: us-central1 + DIRECTOR_ORIGIN: https://relay-staging.onorca.dev + CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + LEGACY_C3_IMAGE_DIGEST: sha256:2d0f6e6db2b0eb9d6aba188698de8330f8c30b4e76badfcf0fac3f3eb9508a87 + steps: + - uses: actions/checkout@v4 + + - id: deploy-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - id: capacity-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: access_token + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Require explicit bootstrap confirmation + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: test "${CONFIRMATION}" = "BOOTSTRAP_STAGING_CAPACITY" + + - name: Read and verify reviewed 600/60 topology + shell: bash + run: | + node dev/scripts/infra.mjs init --env staging + terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ + > "${RUNNER_TEMP}/relay-gce-state.json" + DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars \ + <<< 'local.relay_director_cells_json' | jq -r '.')" + DESIRED_IMAGES_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars \ + <<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.image })' \ + | jq -r '.')" + DESIRED_ZONES_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars \ + <<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.zone })' \ + | jq -r '.')" + DESIRED_TARGET_SIZES_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars \ + <<< 'jsonencode(local.relay_gce_cell_target_sizes)' | jq -r '.')" + jq -e \ + --argjson images "${DESIRED_IMAGES_JSON}" \ + --argjson target_sizes "${DESIRED_TARGET_SIZES_JSON}" \ + '([.[] | select(.id == "staging-gce-c2")] | length == 1) and + ([.[] | select(.id == "staging-gce-c3")] | length == 1) and + (any(.[]; .id == "staging-gce-c2" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and + (any(.[]; .id == "staging-gce-c3" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and + ($images["staging-gce-c2"] == $images["staging-gce-c3"]) and + ($target_sizes["staging-gce-c2"] == 1) and + ($target_sizes["staging-gce-c3"] == 1)' \ + <<< "${DESIRED_CELLS_JSON}" >/dev/null + jq \ + --argjson desired "${DESIRED_CELLS_JSON}" \ + --argjson images "${DESIRED_IMAGES_JSON}" \ + --argjson zones "${DESIRED_ZONES_JSON}" \ + '($desired | map({key: .id, value: .}) | from_entries) as $cells | + to_entries | map(. as $entry | { + key: $entry.key, + value: ($entry.value + { + origin: $cells[$entry.key].url, + image: $images[$entry.key], + zone: $zones[$entry.key], + connection_hard_cap: $cells[$entry.key].connectionHardCap, + connection_unobserved_bound: $cells[$entry.key].connectionUnobservedBound + }) + }) | from_entries' \ + "${RUNNER_TEMP}/relay-gce-state.json" \ + > "${RUNNER_TEMP}/relay-gce-topology.json" + ACTIVE_REVISION="$(gcloud run services describe orca-cloud-relay-staging \ + --project "${GCP_PROJECT_ID}" \ + --region us-central1 \ + --format=json \ + | jq -r ' + [.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" + test -n "${ACTIVE_REVISION}" + DESIRED_IMAGE="$(jq -r '.["staging-gce-c2"]' <<< "${DESIRED_IMAGES_JSON}")" + gcloud run revisions describe "${ACTIVE_REVISION}" \ + --project "${GCP_PROJECT_ID}" \ + --region us-central1 \ + --format=json \ + | jq -e \ + --arg image "${DESIRED_IMAGE}" \ + --arg capacity_service_account "${CAPACITY_SERVICE_ACCOUNT}" \ + '(.spec.containers[0].image == $image) and + any(.spec.containers[0].env[]?; + .name == "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" and + .value == $capacity_service_account)' >/dev/null + CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format=json \ + | jq -cer ' + [.spec.containers[0].env[]? | + select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | + if length == 1 then .[0] | fromjson else error("missing director topology") end')" + jq -e --argjson desired "${DESIRED_CELLS_JSON}" ' + def without_bootstrap_capacity: + map(if .id == "staging-gce-c2" or .id == "staging-gce-c3" + then del(.connectionHardCap, .connectionUnobservedBound) + else . end); + without_bootstrap_capacity == ($desired | without_bootstrap_capacity) + ' <<< "${CURRENT_CELLS_JSON}" >/dev/null + + - name: Bootstrap C2 then C3 + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + + topology="${RUNNER_TEMP}/relay-gce-topology.json" + + fixed_one_instance_name() { + local cell_id="$1" + local mig_name zone + mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")" + zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")" + gcloud compute instance-groups managed list-instances \ + "${mig_name}" \ + --project "${GCP_PROJECT_ID}" \ + --zone "${zone}" \ + --format=json \ + | jq -er ' + if length == 1 and .[0].instanceStatus == "RUNNING" and + .[0].currentAction == "NONE" + then .[0].instance | split("/") | last + else error("legacy cell does not have one stable running instance") end' + } + + fixed_one_instance_id() { + local cell_id="$1" + local instance_name="$2" + local zone + zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")" + gcloud compute instances describe "${instance_name}" \ + --project "${GCP_PROJECT_ID}" \ + --zone "${zone}" \ + --format='value(id)' + } + + write_legacy_metrics() { + local cell_id="$1" + local instance_id="$2" + local after="$3" + local output="$4" + gcloud logging read \ + "resource.type=\"gce_instance\" AND + resource.labels.instance_id=\"${instance_id}\" AND + jsonPayload.event=\"orca_relay_runtime_metrics\" AND + jsonPayload.cellId=\"${cell_id}\" AND + timestamp>=\"${after}\"" \ + --project "${GCP_PROJECT_ID}" \ + --limit 10 \ + --order desc \ + --format json \ + | jq '[.[] | { + timestamp, + cellId: .jsonPayload.cellId, + metricVersion: .jsonPayload.metricVersion, + totalConnections: .jsonPayload.totalConnections, + preAuthConnections: .jsonPayload.preAuthConnections, + controls: .jsonPayload.controls, + splices: .jsonPayload.splices, + pendingSplices: .jsonPayload.pendingSplices, + queuedBytes: .jsonPayload.queuedBytes + }]' > "${output}" + } + + verify_legacy_cell() { + local cell_id="$1" + local admission="$2" + local after="$3" + local expected_instance_id="$4" + local runtime_started_after="${5:-}" + local previous_incarnation_digest="${6:-}" + local hard_cap="${7:-}" + local unobserved_bound="${8:-}" + local capacity_state="${9:-}" + local current_instance current_instance_id metrics origin result + local runtime_start_args=() incarnation_args=() capacity_args=() + current_instance="$(fixed_one_instance_name "${cell_id}")" + current_instance_id="$(fixed_one_instance_id "${cell_id}" "${current_instance}")" + test "${current_instance_id}" = "${expected_instance_id}" + metrics="${RUNNER_TEMP}/${cell_id}-legacy-runtime-metrics.json" + origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" + if test -n "${runtime_started_after}"; then + runtime_start_args=(--runtime-started-after "${runtime_started_after}") + fi + if test -n "${previous_incarnation_digest}"; then + incarnation_args=(--previous-incarnation-digest "${previous_incarnation_digest}") + fi + if test -n "${hard_cap}"; then + capacity_args=(--hard-cap "${hard_cap}" --unobserved-bound "${unobserved_bound}") + fi + if test -n "${capacity_state}"; then + capacity_args+=(--capacity-state "${capacity_state}") + fi + for _attempt in $(seq 1 18); do + write_legacy_metrics \ + "${cell_id}" "${current_instance_id}" "${after}" "${metrics}" + if result="$(node dev/scripts/verify-relay-legacy-bootstrap.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${origin}" \ + --cell-id "${cell_id}" \ + --admission "${admission}" \ + --expected-image-digest "${LEGACY_C3_IMAGE_DIGEST}" \ + --metrics-after "${after}" \ + --metrics-file "${metrics}" \ + "${runtime_start_args[@]}" \ + "${incarnation_args[@]}" \ + "${capacity_args[@]}")"; then + echo "${result}" + return 0 + fi + sleep 10 + done + return 1 + } + + post_admin() { + local origin="$1" + local path="$2" + local body="$3" + curl --fail --silent --show-error \ + --request POST \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "${body}" \ + "${origin}${path}" + } + + runtime_kind() { + local cell_id="$1" + local origin desired_digest digest + origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" + desired_digest="$(jq -r --arg cell "${cell_id}" \ + '.[$cell].image | split("@") | last' "${topology}")" + digest="$(post_admin "${origin}" /v1/admin/runtime-status '{"v":1}' \ + | jq -er '.imageDigest')" + if test "${digest}" = "${LEGACY_C3_IMAGE_DIGEST}"; then + echo legacy + elif test "${digest}" = "${desired_digest}"; then + echo modern + else + echo 'bootstrap cell image is neither legacy nor reviewed' >&2 + return 1 + fi + } + + cell_admission() { + local cell_id="$1" + post_admin "${DIRECTOR_ORIGIN}" /v1/admin/cell-status \ + "$(jq -cn --arg cell "${cell_id}" '{v:1, cellId:$cell}')" \ + | jq -er '.status.admissionState | + if . == "general" or . == "migration-only" then . + else error("bootstrap admission is not recoverable") end' + } + + verify_modern_cell() { + local cell_id="$1" + local admission="$2" + local origin + origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${origin}" \ + --cell-id "${cell_id}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission "${admission}" \ + --draining forbidden \ + --activity allowed + } + + ensure_modern_general() { + local cell_id="$1" + local admission="$2" + verify_modern_cell "${cell_id}" "${admission}" + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${cell_id}" \ + --mode restore \ + --general-cell-ids "${cell_id}" + verify_modern_cell "${cell_id}" general + } + + prepare_legacy_c3_fallback() { + legacy_c3_metrics_boundary="$(node -e \ + 'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')" + legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)" + legacy_c3_instance_id="$(fixed_one_instance_id \ + staging-gce-c3 "${legacy_c3_instance}")" + verify_legacy_cell \ + staging-gce-c3 general \ + "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" + node dev/scripts/probe-relay-legacy-admission.mjs \ + --cell-origin https://c3.relay-staging.onorca.dev + legacy_c3_restart_started_after= + legacy_c3_old_incarnation= + } + + normalize_legacy_c3() { + legacy_pre_boundary="$(node -e \ + 'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')" + legacy_c2_instance="$(fixed_one_instance_name staging-gce-c2)" + legacy_c2_instance_id="$(fixed_one_instance_id \ + staging-gce-c2 "${legacy_c2_instance}")" + verify_legacy_cell \ + staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}" + node dev/scripts/probe-relay-legacy-admission.mjs \ + --cell-origin https://c2.relay-staging.onorca.dev + + legacy_c3_isolated=false + restore_legacy_c3_fallback() { + if test "${legacy_c3_isolated}" = true; then + verify_legacy_cell \ + staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}" + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id staging-gce-c3 \ + --mode restore-fallback \ + --general-cell-ids staging-gce-c2 + fi + } + + trap restore_legacy_c3_fallback EXIT + legacy_c3_isolated=true + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin https://c3.relay-staging.onorca.dev \ + --cell-id staging-gce-c3 \ + --mode isolate + legacy_c3_drain_boundary="$(node -e \ + 'process.stdout.write(new Date().toISOString())')" + legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)" + legacy_c3_instance_id="$(fixed_one_instance_id \ + staging-gce-c3 "${legacy_c3_instance}")" + legacy_c3_drained="$(verify_legacy_cell \ + staging-gce-c3 migration-only \ + "${legacy_c3_drain_boundary}" "${legacy_c3_instance_id}")" + echo "${legacy_c3_drained}" + legacy_c3_old_incarnation="$(jq -er '.incarnationDigest' \ + <<< "${legacy_c3_drained}")" + legacy_c3_restart_started_after="$(node -e \ + 'process.stdout.write(new Date().toISOString())')" + gcloud compute instance-groups managed recreate-instances \ + orca-cloud-staging-relay-gce-c3 \ + --instances "${legacy_c3_instance}" \ + --project "${GCP_PROJECT_ID}" \ + --zone us-central1-a \ + --quiet + gcloud compute instance-groups managed wait-until \ + orca-cloud-staging-relay-gce-c3 \ + --stable \ + --project "${GCP_PROJECT_ID}" \ + --zone us-central1-a \ + --timeout 900 + legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)" + legacy_c3_instance_id="$(fixed_one_instance_id \ + staging-gce-c3 "${legacy_c3_instance}")" + legacy_c3_metrics_boundary="$(node -e \ + 'process.stdout.write(new Date().toISOString())')" + verify_legacy_cell \ + staging-gce-c3 migration-only \ + "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \ + "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}" + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id staging-gce-c3 \ + --mode restore \ + --general-cell-ids staging-gce-c2,staging-gce-c3 + verify_legacy_cell \ + staging-gce-c3 general \ + "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \ + "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}" + legacy_c3_isolated=false + trap - EXIT + } + + roll_cell() ( + local cell_id="$1" + local fallback_cell_id="$2" + local target_kind="$3" + local fallback_kind="$4" + local active_revision cell_origin current_cells_json desired_bound desired_cap + local desired_cells_json director_result image mig_name plan + local fallback_origin plan_changes plan_result restored target_drain_boundary + local target_instance target_instance_id zone + cell_origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" + fallback_origin="$(jq -r --arg cell "${fallback_cell_id}" \ + '.[$cell].origin' "${topology}")" + zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")" + mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")" + image="$(jq -r --arg cell "${cell_id}" '.[$cell].image' "${topology}")" + desired_cap="$(jq -r --arg cell "${cell_id}" \ + '.[$cell].connection_hard_cap' "${topology}")" + desired_bound="$(jq -r --arg cell "${cell_id}" \ + '.[$cell].connection_unobserved_bound' "${topology}")" + plan="${RUNNER_TEMP}/${cell_id}-capacity-bootstrap.tfplan" + restored=false + + restore_fallback() { + if test "${restored}" = false; then + verify_fallback + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${cell_id}" \ + --mode restore-fallback \ + --general-cell-ids "${fallback_cell_id}" + fi + } + + verify_fallback() { + if test "${fallback_kind}" = legacy; then + verify_legacy_cell \ + "${fallback_cell_id}" general \ + "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \ + "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}" + return + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${fallback_origin}" \ + --cell-id "${fallback_cell_id}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed + } + + verify_fallback + trap restore_fallback EXIT + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${cell_origin}" \ + --cell-id "${cell_id}" \ + --mode isolate + target_drain_boundary="$(node -e \ + 'process.stdout.write(new Date().toISOString())')" + if test "${target_kind}" = legacy; then + target_instance="$(fixed_one_instance_name "${cell_id}")" + target_instance_id="$(fixed_one_instance_id "${cell_id}" "${target_instance}")" + verify_legacy_cell \ + "${cell_id}" migration-only \ + "${target_drain_boundary}" "${target_instance_id}" \ + '' '' "${desired_cap}" "${desired_bound}" absent-or-stale + else + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${cell_origin}" \ + --cell-id "${cell_id}" \ + --heartbeat either \ + --admission migration-only \ + --draining required \ + --activity quiescent + fi + + active_revision="$(gcloud run services describe orca-cloud-relay-staging \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format=json \ + | jq -r ' + [.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" + test -n "${active_revision}" + current_cells_json="$(gcloud run revisions describe "${active_revision}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format=json \ + | jq -cer ' + [.spec.containers[0].env[]? | + select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | + if length == 1 then .[0] | fromjson else error("missing director topology") end')" + desired_cells_json="$(jq -ce \ + --arg cell "${cell_id}" \ + --argjson cap "${desired_cap}" \ + --argjson bound "${desired_bound}" \ + 'map(if .id == $cell then . + { + connectionHardCap: $cap, + connectionUnobservedBound: $bound + } else . end)' <<< "${current_cells_json}")" + director_result="$(node dev/scripts/deploy-relay-blue-green.mjs \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --service orca-cloud-relay-staging \ + --image "${image}" \ + --role director \ + --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ + --capacity-cell-id "${cell_id}" \ + --director-cells-json "${desired_cells_json}" \ + --min-instances 0 \ + --prune-revisions true \ + --release-id "bootstrap-${cell_id}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")" + echo "${director_result}" + if test "${target_kind}" = legacy; then + verify_legacy_cell \ + "${cell_id}" migration-only \ + "${target_drain_boundary}" "${target_instance_id}" \ + '' '' "${desired_cap}" "${desired_bound}" + else + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${cell_origin}" \ + --cell-id "${cell_id}" \ + --hard-cap "${desired_cap}" \ + --unobserved-bound "${desired_bound}" \ + --heartbeat either \ + --admission migration-only \ + --draining required \ + --activity quiescent + fi + + terraform -chdir=infra/terraform plan \ + -var-file=environments/staging.tfvars \ + "-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \ + "-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \ + -out="${plan}" + plan_result="$(terraform -chdir=infra/terraform show -json "${plan}" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode bootstrap-cell \ + --cell-id "${cell_id}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --image "${image}" \ + --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")" + echo "${plan_result}" + plan_changes="$(jq -r '.changes' <<< "${plan_result}")" + [[ "${plan_changes}" =~ ^(0|2)$ ]] + if test "${plan_changes}" = 2; then + terraform -chdir=infra/terraform apply -auto-approve "${plan}" + else + target_instance="$(fixed_one_instance_name "${cell_id}")" + gcloud compute instance-groups managed recreate-instances "${mig_name}" \ + --instances "${target_instance}" \ + --project "${GCP_PROJECT_ID}" \ + --zone "${zone}" \ + --quiet + fi + gcloud compute instance-groups managed wait-until "${mig_name}" \ + --stable \ + --project "${GCP_PROJECT_ID}" \ + --zone "${zone}" \ + --timeout 900 + + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${cell_origin}" \ + --cell-id "${cell_id}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission migration-only \ + --draining forbidden \ + --activity allowed + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${cell_id}" \ + --mode restore \ + --general-cell-ids staging-gce-c2,staging-gce-c3 + restored=true + trap - EXIT + ) + + c2_kind="$(runtime_kind staging-gce-c2)" + c3_kind="$(runtime_kind staging-gce-c3)" + c2_admission="$(cell_admission staging-gce-c2)" + c3_admission="$(cell_admission staging-gce-c3)" + bootstrap_phase="$(node dev/scripts/classify-relay-staging-bootstrap.mjs \ + --c2-kind "${c2_kind}" \ + --c2-admission "${c2_admission}" \ + --c3-kind "${c3_kind}" \ + --c3-admission "${c3_admission}")" + jq -cn --arg phase "${bootstrap_phase}" \ + '{event:"relay_staging_bootstrap_phase", phase:$phase}' + + case "${bootstrap_phase}" in + normalize-and-roll-both) + normalize_legacy_c3 + roll_cell staging-gce-c2 staging-gce-c3 legacy legacy + roll_cell staging-gce-c3 staging-gce-c2 legacy modern + ;; + resume-c2-then-c3) + prepare_legacy_c3_fallback + roll_cell staging-gce-c2 staging-gce-c3 legacy legacy + roll_cell staging-gce-c3 staging-gce-c2 legacy modern + ;; + roll-c2) + ensure_modern_general staging-gce-c3 "${c3_admission}" + roll_cell staging-gce-c2 staging-gce-c3 legacy modern + ;; + roll-c3) + ensure_modern_general staging-gce-c2 "${c2_admission}" + roll_cell staging-gce-c3 staging-gce-c2 legacy modern + ;; + complete) + ensure_modern_general staging-gce-c2 "${c2_admission}" + ensure_modern_general staging-gce-c3 "${c3_admission}" + ;; + *) + echo 'unsupported staging bootstrap phase' >&2 + exit 1 + ;; + esac + + - name: Verify both bootstrapped cells + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + for number in 2 3; do + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "https://c${number}.relay-staging.onorca.dev" \ + --cell-id "staging-gce-c${number}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed + done diff --git a/.github/workflows/cloud-deploy-relay-asia-topology.yml b/.github/workflows/cloud-deploy-relay-asia-topology.yml new file mode 100644 index 00000000000..f15fc5ae0e2 --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-asia-topology.yml @@ -0,0 +1,245 @@ +name: Deploy Relay Asia Topology + +on: + workflow_dispatch: + inputs: + environment: + description: Target Relay environment + required: true + type: choice + options: [staging, production] + mode: + description: Validate a saved plan or apply that exact plan + required: true + default: plan + type: choice + options: [plan, apply] + cell-ids: + description: Exact reviewed comma-separated Asia cell set + required: true + type: string + image: + description: Full environment Relay image pinned by sha256 digest + required: true + type: string + confirmation: + description: Enter APPLY_RELAY_ASIA_TOPOLOGY for apply mode + required: false + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }} + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + topology: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 30 + environment: ${{ inputs.environment }} + env: + DEPLOY_MODE: ${{ inputs.mode }} + TARGET_ENVIRONMENT: ${{ inputs.environment }} + TARGET_CELL_IDS: ${{ inputs.cell-ids }} + TARGET_IMAGE: ${{ inputs.image }} + TARGET_REGION: asia-east2 + GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }} + CLOUD_SQL_INSTANCE: ${{ inputs.environment == 'production' && 'orca-cloud-auth-db' || 'orca-cloud-staging-auth-db' }} + VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360 + VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17 + TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }} + TF_VARS: ${{ inputs.environment == 'production' && 'environments/production.tfvars' || 'environments/staging.tfvars' }} + TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }} + TOPOLOGY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT }} + steps: + - uses: actions/checkout@v4 + + - name: Validate the reviewed request before authentication + shell: bash + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + set -euo pipefail + test -n "${TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER}" + test -n "${TOPOLOGY_SERVICE_ACCOUNT}" + case "${TARGET_ENVIRONMENT}:${TARGET_CELL_IDS}" in + staging:staging-gce-c4) ;; + production:production-gce-c27,production-gce-c28,production-gce-c29) ;; + *) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;; + esac + [[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]] + if test "${DEPLOY_MODE}" = apply; then + test "${CONFIRMATION}" = APPLY_RELAY_ASIA_TOPOLOGY + else + test "${DEPLOY_MODE}" = plan + test -z "${CONFIRMATION}" + fi + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.15.8 + terraform_wrapper: false + + - uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ env.TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ env.TOPOLOGY_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }} + object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }} + + - name: Require the checked Cloud SQL connection budget + shell: bash + run: | + set -euo pipefail + budget="$(node dev/scripts/relay-cloud-sql-connection-budget.mjs)" + checked_max="$(jq -er '.maxConnections' <<< "${budget}")" + jq -e '.withinBudget == true' <<< "${budget}" >/dev/null + if test "${TARGET_ENVIRONMENT}" = production; then + instance="$(gcloud sql instances describe "${CLOUD_SQL_INSTANCE}" \ + --project "${GCP_PROJECT_ID}" --format=json)" + live_flag="$(jq -er '[.settings.databaseFlags[]? | + select(.name == "max_connections") | .value] | + if length <= 1 then (.[0] // "") else error("duplicate max_connections flags") end' \ + <<< "${instance}")" + if test -n "${live_flag}"; then + live_max="${live_flag}" + live_source=explicit-flag + else + # The verified production database uses Cloud SQL's 400-connection + # default for this exact shape; fail closed if its shape changes. + test "$(jq -er '.settings.tier' <<< "${instance}")" = \ + "${VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER}" + test "$(jq -er '.databaseVersion' <<< "${instance}")" = \ + "${VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION}" + live_max=400 + live_source=verified-shape-default + fi + test "${live_max}" = "${checked_max}" + else + live_max="not-read-for-staging" + live_source=not-read-for-staging + fi + { + echo "### Relay Cloud SQL connection budget" + echo "- Checked maximum: ${checked_max}" + echo "- Configured maximum: $(jq -er '.configuredMaximum' <<< "${budget}")" + echo "- Rollout operating maximum: $(jq -er '.operatingMaximum' <<< "${budget}")" + echo "- Explicit reserve: $(jq -er '.explicitReserve' <<< "${budget}")" + echo "- Production live max_connections: ${live_max}" + echo "- Production live maximum source: ${live_source}" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Initialize the exact environment state + run: terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}" + + - id: targets + name: Build the exact additive target set + shell: bash + run: | + set -euo pipefail + file="${RUNNER_TEMP}/relay-asia-targets" + : > "${file}" + printf '%s\n' \ + '-target=google_compute_subnetwork.relay_gce_additional["asia-east2"]' \ + '-target=google_compute_router.relay_gce_additional["asia-east2"]' \ + '-target=google_compute_router_nat.relay_gce_additional["asia-east2"]' \ + '-target=google_compute_url_map.relay_gce[0]' >> "${file}" + IFS=, read -ra cells <<< "${TARGET_CELL_IDS}" + for cell_id in "${cells[@]}"; do + printf '%s\n' \ + "-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \ + "-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \ + "-target=google_compute_backend_service.relay_gce_cell[\"${cell_id}\"]" >> "${file}" + done + echo "file=${file}" >> "${GITHUB_OUTPUT}" + + - name: Create and validate the saved topology plan + id: plan + shell: bash + run: | + set -euo pipefail + plan="${RUNNER_TEMP}/relay-asia-topology.tfplan" + plan_json="${RUNNER_TEMP}/relay-asia-topology.json" + mapfile -t targets < "${{ steps.targets.outputs.file }}" + terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \ + -var-file="${TF_VARS}" \ + -var manage_artifact_dns=false \ + "${targets[@]}" -out="${plan}" + terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}" + committed="${RUNNER_TEMP}/relay-committed-asia-topology.json" + jq -e '{ + relay_gce_cells: .variables.relay_gce_cells.value, + relay_gce_additional_region_subnetwork_cidrs: + .variables.relay_gce_additional_region_subnetwork_cidrs.value + }' "${plan_json}" > "${committed}" + node dev/scripts/prepare-relay-asia-topology-input.mjs \ + --existing-json "${committed}" \ + --environment "${TARGET_ENVIRONMENT}" \ + --cell-ids "${TARGET_CELL_IDS}" \ + --image "${TARGET_IMAGE}" + result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \ + --plan-json "${plan_json}" \ + --environment "${TARGET_ENVIRONMENT}" \ + --cell-ids "${TARGET_CELL_IDS}" \ + --region "${TARGET_REGION}" \ + --image "${TARGET_IMAGE}")" + changes="$(jq -er '.changes' <<< "${result}")" + digest="$(sha256sum "${plan}" | awk '{print $1}')" + echo "plan=${plan}" >> "${GITHUB_OUTPUT}" + echo "changes=${changes}" >> "${GITHUB_OUTPUT}" + { + echo "### Relay Asia topology saved plan" + echo "- Environment: ${TARGET_ENVIRONMENT}" + echo "- Cells: ${TARGET_CELL_IDS}" + echo "- Region: ${TARGET_REGION}" + echo "- Mutating resources: ${changes}" + echo "- Saved-plan SHA-256: ${digest}" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Apply only the validated saved plan + if: ${{ inputs.mode == 'apply' }} + run: terraform -chdir=infra/terraform apply -input=false -auto-approve "${{ steps.plan.outputs.plan }}" + + - name: Prove the exact topology targets converged + if: ${{ inputs.mode == 'apply' }} + shell: bash + run: | + set -euo pipefail + mapfile -t targets < "${{ steps.targets.outputs.file }}" + plan="${RUNNER_TEMP}/relay-asia-topology-readback.tfplan" + plan_json="${RUNNER_TEMP}/relay-asia-topology-readback.json" + terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \ + -var-file="${TF_VARS}" \ + -var manage_artifact_dns=false \ + "${targets[@]}" -out="${plan}" + terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}" + result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \ + --plan-json "${plan_json}" \ + --environment "${TARGET_ENVIRONMENT}" \ + --cell-ids "${TARGET_CELL_IDS}" \ + --region "${TARGET_REGION}" \ + --image "${TARGET_IMAGE}")" + test "$(jq -er '.changes' <<< "${result}")" = 0 + + - name: Record the required selector-safe next step + if: ${{ inputs.mode == 'apply' }} + run: | + { + echo "### Required next step" + echo "The VMs are not eligible for ordinary placement yet." + echo "Register the exact new cells atomically as migration-only before any director configuration lists them." + echo "Rollback is migration-only admission; do not destroy the Asia network on rollout day." + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-deploy-relay-fence-broker.yml b/.github/workflows/cloud-deploy-relay-fence-broker.yml new file mode 100644 index 00000000000..624bac69a88 --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-fence-broker.yml @@ -0,0 +1,90 @@ +name: Deploy Relay Fence Broker + +on: + workflow_dispatch: + inputs: + image-digest: + description: Immutable broker image digest built from this main commit + required: true + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + deploy: + if: >- + ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && + github.ref == 'refs/heads/main' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} + SERVICE_NAME: orca-cloud-relay-fence + IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay-fence-broker + IMAGE_DIGEST: ${{ inputs.image-digest }} + steps: + - uses: actions/checkout@v4 + + - uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + + - name: Resolve exact-commit broker image + run: | + [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] + IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}" + SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \ + --format='value(image_summary.digest)')" + test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}" + TAGS="$(gcloud artifacts docker tags list "${IMAGE_REPOSITORY}" \ + --filter="version:${IMAGE_DIGEST}" \ + --format=json)" + jq -e --arg tag "/tags/sha-${GITHUB_SHA}" \ + 'any(.[]; .tag | endswith($tag))' <<< "${TAGS}" + echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}" + + - name: Deploy broker image only + run: | + gcloud run services update "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --image "${IMAGE}" \ + --quiet + + - name: Verify ready singleton revision + run: | + SERVICE="$(gcloud run services describe "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format=json)" + jq -e '.status.conditions[] | select(.type == "Ready" and .status == "True")' \ + <<< "${SERVICE}" + REVISION="$(jq -r \ + '[.status.traffic[] | select((.percent // 0) == 100)] | + if length == 1 then .[0].revisionName // empty else empty end' \ + <<< "${SERVICE}")" + test -n "${REVISION}" + SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format='value(spec.containers[0].image)')" + test "${SERVED_IMAGE}" = "${IMAGE}" diff --git a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml new file mode 100644 index 00000000000..a98f910ada4 --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml @@ -0,0 +1,822 @@ +name: Deploy Relay Production Capacity Job + +on: + workflow_call: + inputs: + mode: + required: true + type: string + target-cell-id: + required: true + type: string + confirmation: + required: true + type: string + monitor-run-id: + required: true + type: string + monitor-run-attempt: + required: true + type: string + evidence-mode: + required: true + type: string + wave-cell-ids: + required: true + type: string + wave-index: + required: true + type: string + source-wave-run-id: + required: true + type: string + +permissions: + actions: read + contents: read + id-token: write + +defaults: + run: + working-directory: cloud + +jobs: + capacity: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 75 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} + DIRECTOR_SERVICE_NAME: orca-cloud-relay + DIRECTOR_ORIGIN: https://relay.onorca.dev + TARGET_CELL_ID: ${{ inputs.target-cell-id }} + CAPACITY_CELL_IDS: production-gce-c7,production-gce-c8,production-gce-c9,production-gce-c10,production-gce-c13,production-gce-c14,production-gce-c15,production-gce-c16,production-gce-c19,production-gce-c20,production-gce-c21,production-gce-c22,production-gce-c23,production-gce-c24,production-gce-c25,production-gce-c26 + PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d + COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73 + COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f + CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + DEPLOY_MODE: ${{ inputs.mode }} + EVIDENCE_MODE: ${{ inputs.evidence-mode }} + MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} + MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} + WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }} + WAVE_INDEX: ${{ inputs.wave-index }} + SOURCE_WAVE_RUN_ID: ${{ inputs.source-wave-run-id }} + steps: + - name: Require exact reusable-workflow invocation + working-directory: . + run: | + [[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]] + if test "${EVIDENCE_MODE}" = continuation; then + test "${DEPLOY_MODE}" = apply + [[ "${WAVE_INDEX}" =~ ^[0-3]$ ]] + test "${WAVE_CELL_IDS}" != none + test "${SOURCE_WAVE_RUN_ID}" = none + elif test "${EVIDENCE_MODE}" = resume; then + test "${DEPLOY_MODE}" = apply + test "${WAVE_INDEX}" = resume + test "${WAVE_CELL_IDS}" != none + [[ "${SOURCE_WAVE_RUN_ID}" =~ ^[0-9]+$ ]] + else + test "${EVIDENCE_MODE}" = single + test "${WAVE_CELL_IDS}" = none + test "${WAVE_INDEX}" = 0 + test "${SOURCE_WAVE_RUN_ID}" = none + fi + + - name: Require production workflow configuration + working-directory: . + env: + DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + CAPACITY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + run: | + test -n "${GCP_REGION}" + test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}" + test -n "${DEPLOY_SERVICE_ACCOUNT}" + test -n "${CAPACITY_WORKLOAD_IDENTITY_PROVIDER}" + test -n "${CAPACITY_SERVICE_ACCOUNT}" + + - uses: actions/checkout@v4 + + - id: resume-provenance + if: ${{ inputs.evidence-mode == 'resume' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + test "${MONITOR_RUN_ATTEMPT}" = 1 + case "${MONITOR_RUN_ID}:${SOURCE_WAVE_RUN_ID}:${WAVE_CELL_IDS}:${TARGET_CELL_ID}" in + 31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13) + EXPECTED_SHA=a917e8e1fc1a2654e8cb81ba39b57733ec56be9c + EXPECTED_SOURCE_ATTEMPT=1 + ;; + 31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10) + EXPECTED_SHA=6082e9ca89a918ca51f0c87db003f5e8805b64b7 + EXPECTED_SOURCE_ATTEMPT=1 + ;; + 31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8) + EXPECTED_SHA=e59958130c9d9b7a6cd805df2678d08997842c7c + EXPECTED_SOURCE_ATTEMPT=2 + ;; + *) exit 1 ;; + esac + MONITOR_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${MONITOR_RUN_ID}" \ + --jq 'select(.name == "Monitor Relay Production" and + .path == ".github/workflows/cloud-monitor-relay-production.yml" and + .head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and + .event == "workflow_dispatch" and .conclusion == "success" and .run_attempt == 1) | + .head_sha')" + SOURCE_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \ + --jq 'select(.name == "Deploy Relay Production Capacity" and + .path == ".github/workflows/cloud-deploy-relay-production-capacity.yml" and + .head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and + .event == "workflow_dispatch" and .conclusion == "failure") | + .head_sha')" + SOURCE_ATTEMPT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \ + --jq '.run_attempt')" + [[ "${MONITOR_SHA}" =~ ^[0-9a-f]{40}$ ]] + test "${MONITOR_SHA}" = "${EXPECTED_SHA}" + test "${SOURCE_SHA}" = "${MONITOR_SHA}" + test "${SOURCE_ATTEMPT}" = "${EXPECTED_SOURCE_ATTEMPT}" + echo "commit-sha=${MONITOR_SHA}" >> "${GITHUB_OUTPUT}" + + - name: Require fresh dry-run evidence reference + if: ${{ inputs.mode == 'apply' }} + run: | + [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] + [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + + - name: Download private dry-run evidence + if: ${{ inputs.mode == 'apply' }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-monitor-evidence + github-token: ${{ github.token }} + run-id: ${{ inputs.monitor-run-id }} + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: cloud/pnpm-lock.yaml + + - run: pnpm install --frozen-lockfile + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Verify dry-run artifact before cloud authentication + if: ${{ inputs.mode == 'apply' }} + run: | + EVIDENCE_COMMIT_SHA="${GITHUB_SHA}" + if test "${EVIDENCE_MODE:-single}" = resume; then + EVIDENCE_COMMIT_SHA="${{ steps.resume-provenance.outputs.commit-sha }}" + fi + node dev/scripts/relay-monitor-evidence.mjs verify-restore \ + --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${EVIDENCE_COMMIT_SHA}" \ + --mode dry-run + + - name: Reject previously consumed dry-run evidence + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + COUNT="$(gh api \ + "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ + --jq '.total_count')" + test "${COUNT}" = "0" + + - name: Download this workflow's wave authority + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-wave-authority + github-token: ${{ github.token }} + run-id: ${{ github.run_id }} + + - name: Download the failed wave authority for resume + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-wave-authority + github-token: ${{ github.token }} + run-id: ${{ inputs.source-wave-run-id }} + + - name: Require wave evidence consumed by this workflow + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${GITHUB_RUN_ID}" + + - name: Require wave evidence consumed by the failed source workflow + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${SOURCE_WAVE_RUN_ID}" + + - id: deploy-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + release: 'false' + + - name: Require exact mutation confirmation + if: ${{ inputs.mode != 'verify' }} + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if test "${EVIDENCE_MODE:-single}" = resume; then + test "${CONFIRMATION}" = "RESUME_SELECTED_CELL_TO_1000 ${TARGET_CELL_ID}" + elif test "${DEPLOY_MODE}" = apply; then + test "${CONFIRMATION}" = "RAISE_SELECTED_CELL_TO_1000" + else + test "${CONFIRMATION}" = "ROLL_BACK_SELECTED_CELL_TO_600 ${TARGET_CELL_ID}" + fi + + - name: Initialize the exact production backend + run: node dev/scripts/infra.mjs init --env production + + - name: Build the exact selected-cell configuration + shell: bash + run: | + if test "${DEPLOY_MODE}" = rollback; then + TARGET_HARD_CAP=600 + else + TARGET_HARD_CAP=1000 + fi + TARGET_UNOBSERVED_BOUND=60 + TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}" + [[ "${TARGET_HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]] + CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev" + CELLS_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/production.tfvars \ + -var manage_artifact_dns=false \ + <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" + OVERRIDE_CELLS_JSON="$(jq -ce \ + --arg cell "${TARGET_CELL_ID}" \ + --argjson cap "${TARGET_HARD_CAP}" \ + --argjson bound "${TARGET_UNOBSERVED_BOUND}" \ + '.[$cell].connection_hard_cap = $cap | + .[$cell].connection_unobserved_bound = $bound' \ + <<< "${CELLS_JSON}")" + jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \ + '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-capacity.tfvars.json" + BASE_CELLS_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/production.tfvars \ + -var manage_artifact_dns=false \ + <<< 'local.relay_director_cells_json' | jq -er '.')" + IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image" + ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone" + DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \ + -var-file=environments/production.tfvars \ + -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \ + -var manage_artifact_dns=false \ + <<< "${IMAGE_EXPRESSION}" | jq -r '.')" + TARGET_ZONE="$(terraform -chdir=infra/terraform console \ + -var-file=environments/production.tfvars \ + -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \ + -var manage_artifact_dns=false \ + <<< "${ZONE_EXPRESSION}" | jq -r '.')" + MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ + | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')" + jq -e --arg cell "${TARGET_CELL_ID}" \ + 'any(.[]; .id == $cell and .connectionHardCap == 1000 and + .connectionUnobservedBound == 60)' \ + <<< "${BASE_CELLS_JSON}" >/dev/null + [[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]] + DESIRED_IMAGE_DIGEST="${DESIRED_IMAGE##*@}" + [[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]] + test "${MIG_NAME}" = "orca-cloud-relay-gce-${TARGET_HOSTNAME}" + { + echo "CELL_ORIGIN=${CELL_ORIGIN}" + echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}" + echo "TARGET_HARD_CAP=${TARGET_HARD_CAP}" + echo "TARGET_UNOBSERVED_BOUND=${TARGET_UNOBSERVED_BOUND}" + echo "DESIRED_IMAGE=${DESIRED_IMAGE}" + echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}" + echo "TARGET_ZONE=${TARGET_ZONE}" + echo "MIG_NAME=${MIG_NAME}" + echo "BASE_CELLS_JSON=${BASE_CELLS_JSON}" + } >> "${GITHUB_ENV}" + + - name: Require the exact compatible production image and topology + shell: bash + run: | + SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" + ACTIVE_REVISION="$(jq -r \ + '[.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \ + <<< "${SERVICE_JSON}")" + test -n "${ACTIVE_REVISION}" + ACTIVE_REVISION_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" + ACTIVE_IMAGE="$(jq -er '.spec.containers[0].image' <<< "${ACTIVE_REVISION_JSON}")" + ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}" + if test "${ACTIVE_IMAGE}" != "${DESIRED_IMAGE}"; then + test "${ACTIVE_IMAGE_DIGEST}" = "${COMPATIBLE_DIRECTOR_IMAGE_DIGEST}" + test "${DESIRED_IMAGE_DIGEST}" = "${COMPATIBLE_CELL_IMAGE_DIGEST}" + fi + CURRENT_CELLS_JSON="$(jq -cer '[.spec.containers[0].env[]? | + select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | + if length == 1 then .[0] | fromjson else error("missing director topology") end' \ + <<< "${ACTIVE_REVISION_JSON}")" + CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON="$( + node dev/scripts/read-relay-production-capacity-identity.mjs \ + <<< "${ACTIVE_REVISION_JSON}" + )" + CLASSIFICATION="$(jq -nc \ + --argjson baseCells "${BASE_CELLS_JSON}" \ + --argjson currentCells "${CURRENT_CELLS_JSON}" \ + --arg capacityCellIds "${CAPACITY_CELL_IDS}" \ + --arg targetCellId "${TARGET_CELL_ID}" \ + --argjson targetHardCap "${TARGET_HARD_CAP}" \ + --argjson currentCapacityServiceAccount \ + "${CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON}" \ + '{baseCells:$baseCells, currentCells:$currentCells, + capacityCellIds:($capacityCellIds | split(",")), + targetCellId:$targetCellId, targetHardCap:$targetHardCap, + currentCapacityServiceAccount:$currentCapacityServiceAccount}' \ + | node dev/scripts/classify-relay-production-capacity-director.mjs \ + --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")" + TOPOLOGY_PHASE="$(jq -er '.topologyPhase' <<< "${CLASSIFICATION}")" + DESIRED_CELLS_JSON="$(jq -cer '.desiredCells' <<< "${CLASSIFICATION}")" + DIRECTOR_READY="$(jq -er \ + 'if (.directorReady | type) == "boolean" then + (.directorReady | tostring) + else error("invalid directorReady classification") end' \ + <<< "${CLASSIFICATION}")" + { + echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" + echo "TOPOLOGY_PHASE=${TOPOLOGY_PHASE}" + echo "DIRECTOR_READY=${DIRECTOR_READY}" + echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" + } >> "${GITHUB_ENV}" + + - name: Require the exact wave predecessor topology + if: ${{ inputs.mode == 'apply' && (inputs.evidence-mode == 'continuation' || inputs.evidence-mode == 'resume') }} + run: test "${TOPOLOGY_PHASE}" = predecessor + + - name: Verify fresh dry-run evidence against the live selector + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ + --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run \ + --mutation-mode capacity-transition \ + --source-cell-id "${TARGET_CELL_ID}" \ + --director-origin "${DIRECTOR_ORIGIN}" + + - name: Recheck every live safety signal + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + pnpm incident:relay-preflight -- \ + --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" + + - name: Recheck exact wave state and every live safety signal + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + node dev/scripts/relay-production-capacity-wave.mjs build-preflight \ + --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \ + --wave-cell-ids "${WAVE_CELL_IDS}" \ + --wave-index "${WAVE_INDEX}" \ + --target-cell-id "${TARGET_CELL_ID}" \ + --output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" + RETRY_ARGS=() + if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi + pnpm incident:relay-preflight -- \ + --state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \ + "${RETRY_ARGS[@]}" + + - name: Recheck exact isolated resume state and every live safety signal + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + node dev/scripts/relay-production-capacity-wave.mjs build-resume-preflight \ + --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \ + --wave-cell-ids "${WAVE_CELL_IDS}" \ + --target-cell-id "${TARGET_CELL_ID}" \ + --output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" + pnpm incident:relay-preflight -- \ + --state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \ + --retry-freshness + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission migration-only \ + --draining required \ + --activity allowed \ + --runtime required \ + --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" + + - name: Consume the single-use dry-run evidence + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" + printf '%s\n' "${GITHUB_RUN_ID}" \ + > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" + + - name: Publish the consumed-evidence marker + if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} + uses: actions/upload-artifact@v4 + with: + name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + retention-days: 90 + if-no-files-found: error + + - name: Verify current selected-cell capacity + if: ${{ inputs.mode == 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + CURRENT_CAP="${TARGET_HARD_CAP}" + CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}" + if test "${TOPOLOGY_PHASE}" = predecessor; then + CURRENT_CAP=600 + CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${CURRENT_CAP}" \ + --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed \ + --expected-image-digests "${CURRENT_IMAGE_DIGEST}" + + - name: Arm fail-closed mutation cleanup + if: ${{ inputs.mode != 'verify' }} + run: echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" + + - name: Reversibly isolate only the selected cell + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + test "${MUTATION_STARTED:-false}" = true || exit 0 + node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode isolate + + - name: Drain the selected cell or prove an offline rollback + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + if node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode drain; then + echo "OFFLINE_ROLLBACK=false" >> "${GITHUB_ENV}" + elif test "${DEPLOY_MODE}" = rollback; then + echo "OFFLINE_ROLLBACK=true" >> "${GITHUB_ENV}" + else + exit 1 + fi + + - id: restart-auth-one + if: ${{ inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - id: restart-gate-one + name: Require restart-safe selected-cell activity + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-one.outputs.id_token }} + run: | + if test "${OFFLINE_ROLLBACK:-false}" = true; then + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --heartbeat stale \ + --admission migration-only \ + --draining either \ + --activity restart-safe \ + --runtime unavailable + echo "settled=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + CURRENT_CAP=600 + if test "${TOPOLOGY_PHASE}" = desired; then + CURRENT_CAP="${TARGET_HARD_CAP}" + elif test "${TARGET_HARD_CAP}" = 600; then + CURRENT_CAP=1000 + fi + GATE_LOG="${RUNNER_TEMP}/relay-capacity-restart-gate-one.log" + set +e + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${CURRENT_CAP}" \ + --unobserved-bound 60 \ + --heartbeat either \ + --admission migration-only \ + --draining required \ + --activity restart-safe \ + --runtime required \ + --timeout-ms 450000 \ + --expected-image-digests \ + "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" \ + 2> "${GATE_LOG}" + GATE_EXIT=$? + set -e + cat "${GATE_LOG}" >&2 + if test "${GATE_EXIT}" = 0; then + echo "settled=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + if test "$(wc -l < "${GATE_LOG}" | tr -d ' ')" = 1 && + grep -Eq '^capacity transition verification timed out: \{.*\}$' "${GATE_LOG}"; then + echo "settled=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + exit "${GATE_EXIT}" + + - id: restart-auth-two + if: ${{ steps.restart-gate-one.outputs.settled == 'false' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Require extended restart-safe selected-cell activity + if: ${{ steps.restart-gate-one.outputs.settled == 'false' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-two.outputs.id_token }} + run: | + CURRENT_CAP=600 + if test "${TOPOLOGY_PHASE}" = desired; then + CURRENT_CAP="${TARGET_HARD_CAP}" + elif test "${TARGET_HARD_CAP}" = 600; then + CURRENT_CAP=1000 + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${CURRENT_CAP}" \ + --unobserved-bound 60 \ + --heartbeat either \ + --admission migration-only \ + --draining required \ + --activity restart-safe \ + --runtime required \ + --timeout-ms 450000 \ + --expected-image-digests \ + "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" + + - name: Deploy only the reviewed director topology + if: ${{ inputs.mode != 'verify' }} + run: | + if test "${DIRECTOR_READY}" = true; then exit 0; fi + RELEASE_ID="capacity-${TARGET_HOSTNAME}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" + node dev/scripts/deploy-relay-blue-green.mjs \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --service "${DIRECTOR_SERVICE_NAME}" \ + --image "${ACTIVE_IMAGE}" \ + --role director \ + --max-instances 5 \ + --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ + --capacity-cell-id "${TARGET_CELL_ID}" \ + --director-cells-json "${DESIRED_CELLS_JSON}" \ + --min-instances 5 \ + --prune-revisions false \ + --release-id "${RELEASE_ID}" + + - id: director-transition-auth + if: ${{ inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Require fail-closed director transition + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.director-transition-auth.outputs.id_token }} + run: | + if test "${OFFLINE_ROLLBACK:-false}" = true; then + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --heartbeat stale \ + --admission migration-only \ + --draining either \ + --activity restart-safe \ + --runtime unavailable + exit 0 + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${TARGET_HARD_CAP}" \ + --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ + --heartbeat either \ + --admission migration-only \ + --draining required \ + --activity restart-safe \ + --runtime required \ + --expected-image-digests \ + "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" + + - id: capacity-auth + if: ${{ inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Plan and apply only the empty selected cell + if: ${{ inputs.mode != 'verify' }} + shell: bash + run: | + terraform -chdir=infra/terraform plan \ + -var-file=environments/production.tfvars \ + -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \ + -var manage_artifact_dns=false \ + "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ + "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ + -out="${RUNNER_TEMP}/relay-capacity-cell.tfplan" + PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \ + "${RUNNER_TEMP}/relay-capacity-cell.tfplan" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode bootstrap-cell \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${TARGET_HARD_CAP}" \ + --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ + --image "${DESIRED_IMAGE}" \ + --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")" + echo "${PLAN_RESULT}" + PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")" + [[ "${PLAN_CHANGES}" =~ ^(0|1|2)$ ]] + if test "${PLAN_CHANGES}" != 0; then + terraform -chdir=infra/terraform apply \ + -auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan" + else + INSTANCE="$(gcloud compute instance-groups managed list-instances \ + "${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \ + --format=json | jq -er 'if length == 1 and + .[0].instanceStatus == "RUNNING" and .[0].currentAction == "NONE" + then .[0].instance | split("/") | last + else error("selected cell is not one stable running instance") end')" + gcloud compute instance-groups managed recreate-instances \ + "${MIG_NAME}" --instances "${INSTANCE}" \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet + fi + gcloud compute instance-groups managed wait-until \ + "${MIG_NAME}" --stable --project "${GCP_PROJECT_ID}" \ + --zone "${TARGET_ZONE}" --timeout 900 + + - id: capacity-transition-auth + if: ${{ inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Verify fresh exact selected-cell heartbeat before admission + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${TARGET_HARD_CAP}" \ + --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission migration-only \ + --draining forbidden \ + --activity allowed \ + --expected-image-digests "${DESIRED_IMAGE_DIGEST}" + + - name: Restore only the selected cell to general admission + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }} + run: | + node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode activate + + - name: Verify the live general selected cell + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${TARGET_HARD_CAP}" \ + --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed \ + --expected-image-digests "${DESIRED_IMAGE_DIGEST}" + + - id: cleanup-auth + if: ${{ failure() && inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Keep the selected cell isolated after a failed mutation + if: ${{ failure() && inputs.mode != 'verify' }} + continue-on-error: true + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }} + run: | + test "${MUTATION_STARTED:-false}" = true || exit 0 + CLEANUP_STATUS=0 + node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode isolate || CLEANUP_STATUS=$? + node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode drain || CLEANUP_STATUS=$? + exit "${CLEANUP_STATUS}" diff --git a/.github/workflows/cloud-deploy-relay-production-capacity.yml b/.github/workflows/cloud-deploy-relay-production-capacity.yml new file mode 100644 index 00000000000..98985eebd6a --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-production-capacity.yml @@ -0,0 +1,353 @@ +name: Deploy Relay Production Capacity + +on: + workflow_dispatch: + inputs: + mode: + description: Verify, change one cell, or raise a sequential wave + required: true + default: verify + type: choice + options: + - verify + - apply + - rollback + - wave-apply + - wave-resume + target-cell-id: + description: Exact serving cell for verify, apply, or rollback + required: true + default: production-gce-c26 + type: choice + options: + - production-gce-c7 + - production-gce-c8 + - production-gce-c9 + - production-gce-c10 + - production-gce-c13 + - production-gce-c14 + - production-gce-c15 + - production-gce-c16 + - production-gce-c19 + - production-gce-c20 + - production-gce-c21 + - production-gce-c22 + - production-gce-c23 + - production-gce-c24 + - production-gce-c25 + - production-gce-c26 + wave-cell-ids: + description: Ordered comma-separated wave of two to four serving cells + required: false + default: none + type: string + confirmation: + description: Enter the exact single-cell or wave confirmation + required: false + type: string + monitor-run-id: + description: Successful fresh dry-run monitor workflow run ID for apply + required: false + type: string + monitor-run-attempt: + description: Exact dry-run monitor workflow attempt for apply + required: false + type: string + source-wave-run-id: + description: Failed wave run that isolated the resume target + required: false + type: string + +permissions: + actions: read + contents: read + id-token: write + +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + single_cell: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode != 'wave-apply' && inputs.mode != 'wave-resume') }} + uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml + with: + mode: ${{ inputs.mode }} + target-cell-id: ${{ inputs.target-cell-id }} + confirmation: ${{ inputs.confirmation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + evidence-mode: single + wave-cell-ids: none + wave-index: '0' + source-wave-run-id: none + secrets: inherit + + resume_cell: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-resume' && github.ref == 'refs/heads/main') }} + uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml + with: + mode: apply + target-cell-id: ${{ inputs.target-cell-id }} + confirmation: ${{ inputs.confirmation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + evidence-mode: resume + wave-cell-ids: ${{ inputs.wave-cell-ids }} + wave-index: resume + source-wave-run-id: ${{ inputs.source-wave-run-id }} + secrets: inherit + + wave_gate: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-apply' && github.ref == 'refs/heads/main') }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 30 + environment: production + outputs: + cells: ${{ steps.wave.outputs.cells }} + env: + DIRECTOR_ORIGIN: https://relay.onorca.dev + MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} + MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} + OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence + PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d + COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f + WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }} + steps: + - name: Require production workflow configuration + working-directory: . + env: + DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + run: | + test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}" + test -n "${DEPLOY_SERVICE_ACCOUNT}" + + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: cloud/pnpm-lock.yaml + + - run: pnpm install --frozen-lockfile + + - id: wave + name: Validate the exact wave request + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + CELLS="$(node dev/scripts/relay-production-capacity-wave.mjs validate \ + --wave-cell-ids "${WAVE_CELL_IDS}" \ + --confirmation "${CONFIRMATION}")" + echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}" + + - name: Require fresh dry-run evidence reference + run: | + [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] + [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + + - name: Download private dry-run evidence + uses: actions/download-artifact@v4 + with: + name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ github.workspace }}/relay-monitor-evidence + github-token: ${{ github.token }} + run-id: ${{ inputs.monitor-run-id }} + + - name: Verify dry-run artifact before cloud authentication + run: | + node dev/scripts/relay-monitor-evidence.mjs verify-restore \ + --directory "${OUTPUT_DIRECTORY}" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run + + - name: Reject previously consumed dry-run evidence + env: + GH_TOKEN: ${{ github.token }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + COUNT="$(gh api \ + "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ + --jq '.total_count')" + test "${COUNT}" = "0" + + - id: deploy-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + release: 'false' + + - name: Verify wave evidence against the live selector + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + FIRST_CELL="$(jq -er '.[0]' <<< '${{ steps.wave.outputs.cells }}')" + node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ + --directory "${OUTPUT_DIRECTORY}" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run \ + --mutation-mode capacity-transition \ + --source-cell-id "${FIRST_CELL}" \ + --director-origin "${DIRECTOR_ORIGIN}" + + - name: Recheck every live safety signal + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + pnpm incident:relay-preflight -- \ + --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" + + - name: Require exact 600/60 predecessor wave cells + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + while read -r CELL_ID; do + HOSTNAME="${CELL_ID#production-gce-}" + [[ "${HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]] + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "https://${HOSTNAME}.relay.onorca.dev" \ + --cell-id "${CELL_ID}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed \ + --expected-image-digests \ + "${PREDECESSOR_IMAGE_DIGEST},${COMPATIBLE_CELL_IMAGE_DIGEST}" + done < <(jq -r '.[]' <<< '${{ steps.wave.outputs.cells }}') + + - name: Consume the single-use dry-run evidence + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" + printf '%s\n' "${GITHUB_RUN_ID}" \ + > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" + + - name: Publish the consumed-evidence marker + uses: actions/upload-artifact@v4 + with: + name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + retention-days: 90 + if-no-files-found: error + + wave_cell_1: + needs: wave_gate + uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml + with: + mode: apply + target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[0] }} + confirmation: RAISE_SELECTED_CELL_TO_1000 + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + evidence-mode: continuation + wave-cell-ids: ${{ inputs.wave-cell-ids }} + wave-index: '0' + source-wave-run-id: none + secrets: inherit + + wave_cell_2: + needs: [wave_gate, wave_cell_1] + uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml + with: + mode: apply + target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[1] }} + confirmation: RAISE_SELECTED_CELL_TO_1000 + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + evidence-mode: continuation + wave-cell-ids: ${{ inputs.wave-cell-ids }} + wave-index: '1' + source-wave-run-id: none + secrets: inherit + + wave_cell_3: + if: ${{ needs.wave_cell_2.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[2] != null }} + needs: [wave_gate, wave_cell_2] + uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml + with: + mode: apply + target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[2] }} + confirmation: RAISE_SELECTED_CELL_TO_1000 + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + evidence-mode: continuation + wave-cell-ids: ${{ inputs.wave-cell-ids }} + wave-index: '2' + source-wave-run-id: none + secrets: inherit + + wave_cell_4: + if: ${{ needs.wave_cell_3.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[3] != null }} + needs: [wave_gate, wave_cell_3] + uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml + with: + mode: apply + target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[3] }} + confirmation: RAISE_SELECTED_CELL_TO_1000 + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + evidence-mode: continuation + wave-cell-ids: ${{ inputs.wave-cell-ids }} + wave-index: '3' + source-wave-run-id: none + secrets: inherit + + # Every wave job re-enters the run's lease with release: 'false'; only this job frees it. + release_lease: + if: always() + needs: + - single_cell + - resume_cell + - wave_gate + - wave_cell_1 + - wave_cell_2 + - wave_cell_3 + - wave_cell_4 + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 10 + environment: production + steps: + - uses: actions/checkout@v4 + + - uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + release: 'true' diff --git a/.github/workflows/cloud-deploy-relay-production-director.yml b/.github/workflows/cloud-deploy-relay-production-director.yml new file mode 100644 index 00000000000..97abe2d227b --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-production-director.yml @@ -0,0 +1,267 @@ +name: Deploy Relay Production Director + +on: + workflow_dispatch: + inputs: + image-digest: + description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)" + required: true + type: string + regional-placement-mode: + description: Preserve the live switch, explicitly enable Asia preference, or force US-first + required: true + default: preserve + type: choice + options: [preserve, enable, disable] + prune-incompatible-revisions: + description: Retain only the newly verified serving and rollback revisions + required: true + default: false + type: boolean + confirmation: + description: Enter the exact confirmation required by a destructive option + required: false + type: string + expected-rehome-generation: + description: Exact durable regional-rehome generation; it must remain disabled + required: true + type: string + bootstrap-runtime-identity: + description: One-time move from the stamped-cell identity to the director identity + required: true + default: false + type: boolean + predecessor-image-digest: + description: Exact immutable serving predecessor digest for the one-time identity bootstrap + required: true + type: string + +permissions: + contents: read + id-token: write + +# Director updates and candidate operations both mutate production relay control state. +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + deploy: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} + DIRECTOR_SERVICE_NAME: orca-cloud-relay + IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay + REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled + IMAGE_DIGEST: ${{ inputs.image-digest }} + REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }} + PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }} + # Floor the served revision must keep, matching relay_min_instances in + # environments/production.tfvars. This gate only fails a bad deploy; Terraform + # still owns the value, and the candidate inherits it from the serving revision. + DIRECTOR_MIN_INSTANCES: 5 + DIRECTOR_MAX_INSTANCES: 5 + DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} + PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT }} + REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain + EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} + BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }} + PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} + steps: + - uses: actions/checkout@v4 + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Resolve immutable production image + shell: bash + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if [[ ! "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "image-digest must be an immutable lowercase sha256 digest" >&2 + exit 1 + fi + IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}" + SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')" + test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}" + [[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]] + [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] + [[ "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]] + [[ "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]] + [[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]] + if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then + [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false + test "${REGIONAL_PLACEMENT_MODE}" = preserve + test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY + elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then + test "${REGIONAL_PLACEMENT_MODE}" = preserve + test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS + elif test "${REGIONAL_PLACEMENT_MODE}" = disable; then + test "${CONFIRMATION}" = FORCE_RELAY_US_FIRST + else + test -z "${CONFIRMATION}" + fi + echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}" + + # Why: the deploy INHERITS the serving revision's floor, so when that revision has + # already lost it the candidate inherits zero, the in-script gate compares zero against + # zero and passes, and the post-deploy check below only notices after traffic moved. + # The documented rollback target is created at minimum instances zero, so promoting it + # arms exactly that. Refuse to inherit a degraded floor rather than latch it. + - name: Require a healthy serving floor before deploying + shell: bash + run: | + SERVING="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -r '[.status.traffic[] | select((.percent // 0) > 0)] + | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" + test -n "${SERVING}" + FLOOR="$(gcloud run revisions describe "${SERVING}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")" + if [[ "${FLOOR:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then + echo "serving revision ${SERVING} holds ${FLOOR:-0} minimum instances," \ + "below ${DIRECTOR_MIN_INSTANCES}; deploying would inherit and latch it." >&2 + echo "Restore the floor first: gcloud run services update ${DIRECTOR_SERVICE_NAME}" \ + "--min-instances=${DIRECTOR_MIN_INSTANCES}" >&2 + exit 1 + fi + CEILING="$(gcloud run revisions describe "${SERVING}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")" + test "${CEILING}" = "${DIRECTOR_MAX_INSTANCES}" + echo "serving revision ${SERVING} holds ${FLOOR} minimum instances" + echo "SERVING_REVISION=${SERVING}" >> "${GITHUB_ENV}" + + # Why: no --min-instances here. The candidate inherits the Terraform-owned + # scaling, and this step ends with 100% traffic on it. Pinning 1 rebuilt the + # per-instance admission shortage that took placement failures to ~70%. + - name: Deploy director blue/green + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + served_version="$(gcloud run revisions describe "${SERVING_REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -r '[.spec.containers[0].env[]? | + select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") | + (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) | + (.version // .key // empty)] | + if length == 1 then .[0] else empty end')" + if [[ "${served_version}" =~ ^[1-9][0-9]*$ ]]; then + current_version="${served_version}" + else + test "${REGIONAL_PLACEMENT_MODE}" = preserve + current_version="$(gcloud secrets versions describe latest \ + --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \ + --format='value(name)' | awk -F/ '{print $NF}')" + [[ "${current_version}" =~ ^[1-9][0-9]*$ ]] + fi + current="$(gcloud secrets versions access "${current_version}" \ + --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")" + [[ "${current}" =~ ^(true|false)$ ]] + case "${REGIONAL_PLACEMENT_MODE}" in + preserve) desired="${current}" ;; + enable) desired=true ;; + disable) desired=false ;; + *) echo "regional-placement-mode is invalid" >&2; exit 1 ;; + esac + if test "${current}" != "${desired}"; then + target_version="$(printf '%s' "${desired}" | gcloud secrets versions add \ + "${REGIONAL_PLACEMENT_SECRET}" --project "${GCP_PROJECT_ID}" --data-file=- \ + --format='value(name)' --quiet | awk -F/ '{print $NF}')" + else + target_version="${current_version}" + fi + [[ "${target_version}" =~ ^[1-9][0-9]*$ ]] + RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" + node dev/scripts/deploy-relay-blue-green.mjs \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --service "${DIRECTOR_SERVICE_NAME}" \ + --image "${IMAGE}" \ + --role director \ + --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ + --predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \ + --bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \ + --predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}" \ + --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ + --rehome-audience "${REHOME_AUDIENCE}" \ + --rehome-control-origin https://relay.onorca.dev \ + --admin-audience https://relay.onorca.dev/v1/admin/drain \ + --expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \ + --max-instances "${DIRECTOR_MAX_INSTANCES}" \ + --prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \ + --release-id "${RELEASE_ID}" \ + --regional-placement-secret-version "${target_version}" + echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}" + echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}" + + - name: Verify served revision and native health + shell: bash + run: | + SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format=json)" + REVISION="$(jq -r '[.status.traffic[] | select((.percent // 0) > 0)] | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' <<< "${SERVICE_JSON}")" + test -n "${REVISION}" + SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format='value(spec.containers[0].image)')" + test "${SERVED_IMAGE}" = "${IMAGE}" + SERVED_REGIONAL_PLACEMENT_SECRET="$(gcloud run revisions describe "${REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -cer '[.spec.containers[0].env[] | + select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") | + (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) | + {secret: (.secret // .name), version: (.version // .key)}] | + if length == 1 then .[0] else error("regional placement secret missing") end')" + test "$(jq -r '.secret' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \ + "${REGIONAL_PLACEMENT_SECRET}" + test "$(jq -r '.version' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \ + "${REGIONAL_PLACEMENT_VERSION}" + test "$(gcloud secrets versions access "${REGIONAL_PLACEMENT_VERSION}" --project "${GCP_PROJECT_ID}" \ + --secret "${REGIONAL_PLACEMENT_SECRET}")" = "${REGIONAL_PLACEMENT_ENABLED}" + # Why: a served revision with no warm-instance floor still passes health and digest + # checks while quietly shrinking per-instance admission capacity. + SERVED_MIN_INSTANCES="$(gcloud run revisions describe "${REVISION}" \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")" + if [[ "${SERVED_MIN_INSTANCES:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then + echo "served revision ${REVISION} holds ${SERVED_MIN_INSTANCES:-0} minimum instances, expected at least ${DIRECTOR_MIN_INSTANCES}" >&2 + exit 1 + fi + SERVED_MAX_INSTANCES="$(gcloud run revisions describe "${REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")" + test "${SERVED_MAX_INSTANCES}" = "${DIRECTOR_MAX_INSTANCES}" + SERVICE_URL="$(jq -r '.status.url' <<< "${SERVICE_JSON}")" + node dev/scripts/smoke-relay.mjs "${SERVICE_URL}" diff --git a/.github/workflows/cloud-deploy-relay-production-multi-target.yml b/.github/workflows/cloud-deploy-relay-production-multi-target.yml new file mode 100644 index 00000000000..8f27f568b50 --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-production-multi-target.yml @@ -0,0 +1,499 @@ +name: Deploy Relay Production Multi-Target + +on: + workflow_dispatch: + inputs: + source-cell-id: + description: Existing Terraform source cell ID + required: true + type: string + target-cell-ids: + description: Comma-separated distinct Terraform target cell IDs + required: true + type: string + general-cell-ids: + description: Comma-separated proven cells that remain eligible for ordinary placement + required: false + type: string + unobserved-connection-bound: + description: Exact worst-case unobserved connection bound proven by the passing load gate + required: false + type: string + failed-target-cell-id: + description: Registered failed target to fence and supersede + required: false + type: string + replacement-target-cell-id: + description: Healthy replacement for registered failed target + required: false + type: string + mode: + description: Preflight/audit are read-only; other modes mutate production + required: true + default: preflight + type: choice + options: + - audit + - preflight + - cutover-admission + - add-migration-cells + - promote-general-cell + - retire-migration-cell + - execute + - recover-forward + - fence-source + - supersede-target + confirmation: + description: Enter CUTOVER_SELECTOR, ADD_MIGRATION_CELLS, PROMOTE_GENERAL_CELL, RETIRE_MIGRATION_CELL, EVACUATE_MULTI, RECOVER_FORWARD, or FENCE_SOURCE + required: false + type: string + selector-attempt-id: + description: Exact durable selector attempt ID for admission mutations + required: false + type: string + monitor-run-id: + description: Successful fresh dry-run monitor workflow run ID + required: false + type: string + monitor-run-attempt: + description: Exact dry-run monitor workflow attempt + required: false + type: string + broker-operation-id: + description: Stable durable broker operation ID for target supersession + required: false + type: string + completed-fence-attempt-id: + description: Exact older completed fence attempt to recover without replay + required: false + type: string + completed-fence-commit: + description: Exact older fence commit bound to the completed attempt + required: false + type: string + completed-fence-operation: + description: Exact DONE Compute resize operation to adopt + required: false + type: string + completed-fence-state-serial: + description: Exact Terraform serial before the completed fence + required: false + type: string + completed-fence-plan-generation: + description: Exact saved-plan object generation + required: false + type: string + completed-fence-state-generation: + description: Exact current Terraform state object generation + required: false + type: string + completed-fence-state-sha256: + description: Exact current Terraform state object SHA-256 + required: false + type: string + expected-lease-generation: + description: Exact live lease generation authorized for conditional takeover + required: false + type: string + expected-lease-operation-id: + description: Exact live lease operation ID authorized for takeover + required: false + type: string + expected-lease-request-digest: + description: Exact live lease request digest authorized for takeover + required: false + type: string + +permissions: + actions: read + contents: read + id-token: write + +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + deploy: + if: >- + ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && + github.ref == 'refs/heads/main' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + DIRECTOR_ORIGIN: https://relay.onorca.dev + ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain + SOURCE_CELL_ID: ${{ inputs.source-cell-id }} + TARGET_CELL_IDS: ${{ inputs.target-cell-ids }} + GENERAL_CELL_IDS: ${{ inputs.general-cell-ids }} + UNOBSERVED_CONNECTION_BOUND: ${{ inputs.unobserved-connection-bound }} + FAILED_TARGET_CELL_ID: ${{ inputs.failed-target-cell-id }} + REPLACEMENT_TARGET_CELL_ID: ${{ inputs.replacement-target-cell-id }} + DEPLOY_MODE: ${{ inputs.mode }} + MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} + MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} + SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }} + BROKER_OPERATION_ID: ${{ inputs.broker-operation-id }} + COMPLETED_FENCE_ATTEMPT_ID: ${{ inputs.completed-fence-attempt-id }} + COMPLETED_FENCE_COMMIT: ${{ inputs.completed-fence-commit }} + COMPLETED_FENCE_OPERATION: ${{ inputs.completed-fence-operation }} + COMPLETED_FENCE_STATE_SERIAL: ${{ inputs.completed-fence-state-serial }} + COMPLETED_FENCE_PLAN_GENERATION: ${{ inputs.completed-fence-plan-generation }} + COMPLETED_FENCE_STATE_GENERATION: ${{ inputs.completed-fence-state-generation }} + COMPLETED_FENCE_STATE_SHA256: ${{ inputs.completed-fence-state-sha256 }} + EXPECTED_LEASE_GENERATION: ${{ inputs.expected-lease-generation }} + EXPECTED_LEASE_OPERATION_ID: ${{ inputs.expected-lease-operation-id }} + EXPECTED_LEASE_REQUEST_DIGEST: ${{ inputs.expected-lease-request-digest }} + steps: + - uses: actions/checkout@v4 + + - name: Require private fence-broker environment + if: >- + ${{ inputs.mode == 'fence-source' || + inputs.mode == 'supersede-target' }} + env: + FENCE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }} + FENCE_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }} + FENCE_BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} + run: | + test -n "${FENCE_WORKLOAD_IDENTITY_PROVIDER}" + test -n "${FENCE_SERVICE_ACCOUNT}" + test -n "${FENCE_BROKER_URI}" + + - name: Reject direct-runner Terraform fence aborts + if: ${{ inputs.mode == 'abort-fence-source' }} + run: | + echo "Terraform fence aborts require a reviewed private-broker recovery path." >&2 + exit 1 + + - name: Require fresh dry-run evidence reference + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + run: | + [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] + [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + + - name: Download private dry-run evidence + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-monitor-evidence + github-token: ${{ github.token }} + run-id: ${{ inputs.monitor-run-id }} + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: cloud/pnpm-lock.yaml + + - run: pnpm install --frozen-lockfile + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Verify dry-run artifact before cloud authentication + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + run: | + node dev/scripts/relay-monitor-evidence.mjs verify-restore \ + --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run + + - name: Reject previously consumed dry-run evidence + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + COUNT="$(gh api \ + "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ + --jq '.total_count')" + test "${COUNT}" = "0" + + - id: google-auth + if: ${{ inputs.mode != 'supersede-target' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + + - name: Require explicit mutation confirmation + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if [[ "${DEPLOY_MODE}" = "cutover-admission" ]]; then + test "${CONFIRMATION}" = "CUTOVER_SELECTOR" + elif [[ "${DEPLOY_MODE}" = "add-migration-cells" ]]; then + test "${CONFIRMATION}" = "ADD_MIGRATION_CELLS" + elif [[ "${DEPLOY_MODE}" = "promote-general-cell" ]]; then + test "${CONFIRMATION}" = "PROMOTE_GENERAL_CELL" + elif [[ "${DEPLOY_MODE}" = "retire-migration-cell" ]]; then + test "${CONFIRMATION}" = "RETIRE_MIGRATION_CELL" + elif [[ "${DEPLOY_MODE}" = "execute" ]]; then + test "${CONFIRMATION}" = "EVACUATE_MULTI" + elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then + test "${CONFIRMATION}" = "RECOVER_FORWARD" + elif [[ "${DEPLOY_MODE}" = "fence-source" ]]; then + test "${CONFIRMATION}" = "FENCE_SOURCE" + elif [[ "${DEPLOY_MODE}" = "supersede-target" ]]; then + test "${CONFIRMATION}" = "SUPERSEDE_TARGET" + elif [[ "${DEPLOY_MODE}" = "abort-fence-source" ]]; then + test "${CONFIRMATION}" = "ABORT_FENCE" + else + test "${CONFIRMATION}" = "FENCE_SOURCE" + fi + + - name: Require exact source-fence broker contract + if: ${{ inputs.mode == 'fence-source' }} + run: | + test "${SOURCE_CELL_ID}" = "production-gce-c3" + test "${TARGET_CELL_IDS}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18" + [[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] + if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then + [[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]] + test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}" + [[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]] + else + test -z "${EXPECTED_LEASE_OPERATION_ID}" + test -z "${EXPECTED_LEASE_REQUEST_DIGEST}" + fi + + - name: Require exact broker cell contract + if: ${{ inputs.mode == 'supersede-target' }} + run: | + test "${SOURCE_CELL_ID}" = "production-gce-c3" + test "${FAILED_TARGET_CELL_ID}" = "production-gce-c12" + test "${REPLACEMENT_TARGET_CELL_ID}" = "production-gce-c13" + test "${TARGET_CELL_IDS}" = "production-gce-c12,production-gce-c13" + if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then + [[ "${COMPLETED_FENCE_ATTEMPT_ID}" =~ ^[0-9a-f-]{36}$ ]] + [[ "${COMPLETED_FENCE_COMMIT}" =~ ^[0-9a-f]{40}$ ]] + [[ "${COMPLETED_FENCE_OPERATION}" =~ ^[A-Za-z0-9._-]{1,256}$ ]] + [[ "${COMPLETED_FENCE_STATE_SERIAL}" =~ ^[0-9]+$ ]] + [[ "${COMPLETED_FENCE_PLAN_GENERATION}" =~ ^[1-9][0-9]*$ ]] + [[ "${COMPLETED_FENCE_STATE_GENERATION}" =~ ^[1-9][0-9]*$ ]] + [[ "${COMPLETED_FENCE_STATE_SHA256}" =~ ^[0-9a-f]{64}$ ]] + test -n "${EXPECTED_LEASE_GENERATION}" + fi + if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then + [[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]] + test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}" + [[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]] + else + test -z "${EXPECTED_LEASE_OPERATION_ID}" + test -z "${EXPECTED_LEASE_REQUEST_DIGEST}" + fi + + - name: Read reviewed Terraform topology + if: ${{ inputs.mode != 'supersede-target' }} + run: | + node dev/scripts/infra.mjs init --env production + terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" + RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)" + DIRECTOR_MIN_INSTANCES="$(terraform -chdir=infra/terraform console \ + -var-file=environments/production.tfvars <<< 'var.relay_min_instances')" + [[ "${DIRECTOR_MIN_INSTANCES}" =~ ^[1-9][0-9]*$ ]] + echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}" + echo "DIRECTOR_MIN_INSTANCES=${DIRECTOR_MIN_INSTANCES}" >> "${GITHUB_ENV}" + + - name: Verify fresh dry-run evidence against live selector + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + SCOPED_RECOVERY_ARGS=() + if [[ ("${DEPLOY_MODE}" = "execute" || + "${DEPLOY_MODE}" = "recover-forward") && + "${SOURCE_CELL_ID}" = "production-gce-c12" ]]; then + SCOPED_RECOVERY_ARGS=( + --scoped-recovery-source-cell-id + production-gce-c3 + ) + fi + node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ + --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run \ + --mutation-mode "${DEPLOY_MODE}" \ + --source-cell-id "${SOURCE_CELL_ID}" \ + "${SCOPED_RECOVERY_ARGS[@]}" \ + --director-origin "${DIRECTOR_ORIGIN}" + + - name: Recheck all live safety signals + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + pnpm incident:relay-preflight -- \ + --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" + + - name: Create single-use dry-run marker + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" + printf '%s\n' "${GITHUB_RUN_ID}" \ + > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" + + - name: Consume dry-run evidence + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} + uses: actions/upload-artifact@v4 + with: + name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + retention-days: 90 + if-no-files-found: error + + - id: google-fence-broker-auth + if: >- + ${{ inputs.mode == 'fence-source' || + inputs.mode == 'supersede-target' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} + id_token_include_email: true + + - name: Invoke private target-supersession broker + if: ${{ inputs.mode == 'supersede-target' }} + env: + BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }} + BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} + run: | + [[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] + if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then + REQUEST="$(jq -cn \ + --arg operationId "${BROKER_OPERATION_ID}" \ + --arg fenceCommit "${GITHUB_SHA}" \ + --arg attemptId "${COMPLETED_FENCE_ATTEMPT_ID}" \ + --arg completedCommit "${COMPLETED_FENCE_COMMIT}" \ + --arg gceOperation "${COMPLETED_FENCE_OPERATION}" \ + --arg stateSerial "${COMPLETED_FENCE_STATE_SERIAL}" \ + --arg planGeneration "${COMPLETED_FENCE_PLAN_GENERATION}" \ + --arg stateGeneration "${COMPLETED_FENCE_STATE_GENERATION}" \ + --arg stateSha256 "${COMPLETED_FENCE_STATE_SHA256}" \ + --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \ + --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \ + --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \ + '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, + completedFenceRecovery:{attemptId:$attemptId,fenceCommit:$completedCommit, + gceOperation:$gceOperation,terraformStateSerial:($stateSerial|tonumber), + planObjectGeneration:$planGeneration, + terraformStateObjectGeneration:$stateGeneration, + terraformStateObjectSha256:$stateSha256}, + expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId, + requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')" + elif [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then + REQUEST="$(jq -cn \ + --arg operationId "${BROKER_OPERATION_ID}" \ + --arg fenceCommit "${GITHUB_SHA}" \ + --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \ + --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \ + --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \ + '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, + expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId, + requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')" + else + REQUEST="$(jq -cn \ + --arg operationId "${BROKER_OPERATION_ID}" \ + --arg fenceCommit "${GITHUB_SHA}" \ + '{v:1,operationId:$operationId,fenceCommit:$fenceCommit,confirmation:"SUPERSEDE_TARGET"}')" + fi + curl --fail-with-body --max-time 1790 \ + --request POST "${BROKER_URI}/v1/supersede-target" \ + --header "Authorization: Bearer ${BROKER_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "${REQUEST}" + + - name: Invoke private source-fence broker + if: ${{ inputs.mode == 'fence-source' }} + env: + BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }} + BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} + run: | + if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then + REQUEST="$(jq -cn \ + --arg operationId "${BROKER_OPERATION_ID}" \ + --arg fenceCommit "${GITHUB_SHA}" \ + --arg targetCellIds "${TARGET_CELL_IDS}" \ + --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \ + --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \ + --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \ + '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, + targetCellIds:($targetCellIds|split(",")), + expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId, + requestDigest:$leaseRequestDigest},confirmation:"FENCE_SOURCE"}')" + else + REQUEST="$(jq -cn \ + --arg operationId "${BROKER_OPERATION_ID}" \ + --arg fenceCommit "${GITHUB_SHA}" \ + --arg targetCellIds "${TARGET_CELL_IDS}" \ + '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, + targetCellIds:($targetCellIds|split(",")),confirmation:"FENCE_SOURCE"}')" + fi + curl --fail-with-body --max-time 1790 \ + --request POST "${BROKER_URI}/v1/fence-source" \ + --header "Authorization: Bearer ${BROKER_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "${REQUEST}" + + - name: Preflight or run multi-target evacuation + if: >- + ${{ inputs.mode != 'fence-source' && + inputs.mode != 'supersede-target' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/deploy-relay-gce-multi-target.mjs \ + --project "${GCP_PROJECT_ID}" \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --admin-audience "${ADMIN_AUDIENCE}" \ + --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \ + --source-cell-id "${SOURCE_CELL_ID}" \ + --target-cell-ids "${TARGET_CELL_IDS}" \ + --general-cell-ids "${GENERAL_CELL_IDS}" \ + --unobserved-connection-bound "${UNOBSERVED_CONNECTION_BOUND}" \ + --director-region "${{ vars.PRODUCTION_GCP_REGION }}" \ + --director-service "orca-cloud-relay" \ + --director-min-instances "${DIRECTOR_MIN_INSTANCES}" \ + --selector-attempt-id "${SELECTOR_ATTEMPT_ID}" \ + --failed-target-cell-id "${FAILED_TARGET_CELL_ID}" \ + --replacement-target-cell-id "${REPLACEMENT_TARGET_CELL_ID}" \ + --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \ + --environment production \ + --fence-commit "${GITHUB_SHA}" \ + --terraform-dir infra/terraform \ + --terraform-var-file environments/production.tfvars \ + --mode "${DEPLOY_MODE}" \ + --connection-ceiling 1000 \ + --minimum-lease-remaining-ms 600000 diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml new file mode 100644 index 00000000000..a8d8c68fafd --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -0,0 +1,645 @@ +name: Deploy Relay Production Same-Cap Job + +on: + workflow_call: + inputs: + mode: { required: true, type: string } + target-cell-id: { required: true, type: string } + target-image-digest: { required: true, type: string } + rollback-image-digest: { required: true, type: string } + target-rehome-protocol: { required: true, type: string } + rollback-rehome-protocol: { required: true, type: string } + expected-selector-generation: { required: true, type: string } + expected-existing-only-cells: { required: true, type: string } + expected-migration-only-cells: { required: true, type: string } + expected-general-cells: { required: true, type: string } + expected-rehome-generation: { required: true, type: string } + monitor-run-id: { required: true, type: string } + monitor-run-attempt: { required: true, type: string } + wave-index: { required: true, type: string } + +permissions: + actions: read + contents: read + id-token: write + +defaults: + run: + working-directory: cloud + +jobs: + rollout: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 75 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} + DIRECTOR_ORIGIN: https://relay.onorca.dev + IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay + TARGET_CELL_ID: ${{ inputs.target-cell-id }} + DEPLOY_MODE: ${{ inputs.mode }} + TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }} + ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }} + TARGET_REHOME_PROTOCOL: ${{ inputs.target-rehome-protocol }} + ROLLBACK_REHOME_PROTOCOL: ${{ inputs.rollback-rehome-protocol }} + EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} + EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }} + EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }} + EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }} + EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} + WAVE_INDEX: ${{ inputs.wave-index }} + MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} + MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} + OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence + steps: + - name: Require exact reusable-workflow configuration + working-directory: . + env: + DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + CAPACITY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} + run: | + [[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]] + [[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + [[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}" + [[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]] + [[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]] + [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] + [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] + [[ "${WAVE_INDEX}" =~ ^[0-3]$ ]] + if test "${DEPLOY_MODE}" = verify; then + EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}" + else + EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION + (2 * WAVE_INDEX)))" + fi + echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" >> "${GITHUB_ENV}" + if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then + echo "mutations are single-dispatch: re-runs replay aged evidence," >&2 + echo "so recover each remaining cell with its own fresh monitor" >&2 + echo "dry-run and canary-apply dispatch instead" >&2 + exit 1 + fi + test -n "${GCP_REGION}" + test -n "${DEPLOY_WIF}" + test -n "${DEPLOY_SERVICE_ACCOUNT}" + test -n "${CAPACITY_WIF}" + test -n "${CAPACITY_SERVICE_ACCOUNT}" + test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" + + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: { package_json_file: cloud/package.json } + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: cloud/pnpm-lock.yaml + + - run: pnpm install --frozen-lockfile + + - uses: hashicorp/setup-terraform@v3 + with: { terraform_wrapper: false } + + - name: Require fresh aggregate monitor evidence reference + if: ${{ inputs.mode != 'verify' }} + run: | + [[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]] + [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + + - name: Download private aggregate monitor evidence + if: ${{ inputs.mode != 'verify' }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ github.workspace }}/relay-monitor-evidence + github-token: ${{ github.token }} + run-id: ${{ inputs.monitor-run-id }} + + - name: Verify monitor evidence provenance + if: ${{ inputs.mode != 'verify' }} + run: | + node dev/scripts/relay-monitor-evidence.mjs verify-authority \ + --directory "${OUTPUT_DIRECTORY}" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run \ + --required-migration-policy strict \ + --wave-index "${WAVE_INDEX}" + + - name: Download this wave's single-use safety authority + if: ${{ inputs.mode != 'verify' }} + uses: actions/download-artifact@v4 + with: + name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-same-cap-monitor-authority + github-token: ${{ github.token }} + run-id: ${{ github.run_id }} + + - name: Require safety evidence consumed by this workflow + if: ${{ inputs.mode != 'verify' }} + run: | + # Mutations are single-dispatch: a fresh dispatch cannot resume a + # partial batch (the canary authority binds the batch-entry selector + # generation), so each remaining cell is recovered by its own fresh + # monitor dry-run and canary-apply dispatch, never by re-running + # aged evidence. + test "${GITHUB_RUN_ATTEMPT}" = 1 + MARKER_NAME="relay-same-cap-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + test "$(< "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}")" = \ + "${GITHUB_RUN_ID}" + + - id: deploy-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + release: 'false' + + - name: Recheck aggregate SQL, pool, reconnect, migration, and selector safety + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + RETRY_ARGS=() + if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi + pnpm incident:relay-preflight -- \ + --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \ + --wave-index "${WAVE_INDEX}" "${RETRY_ARGS[@]}" + + - name: Require durable rehome disabled and exact selector + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode inspect \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-selector-generation "${EFFECTIVE_SELECTOR_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ + --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ + --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \ + | jq -e '.control.enabled == false' >/dev/null + + - name: Initialize the exact production backend + run: node dev/scripts/infra.mjs init --env production + + - name: Resolve immutable same-cap cell configuration + shell: bash + run: | + TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}" + case "${TARGET_HOSTNAME}" in + c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26) + EXPECTED_HARD_CAP=1000 + EXPECTED_REGION=us-central1 + ;; + c27|c28|c29) + EXPECTED_HARD_CAP=3000 + EXPECTED_REGION=asia-east2 + ;; + *) exit 1 ;; + esac + EXPECTED_UNOBSERVED_BOUND=60 + CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev" + CELLS_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/production.tfvars -var manage_artifact_dns=false \ + <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" + SOURCE_CELLS="$(terraform -chdir=infra/terraform console \ + -var-file=environments/production.tfvars -var manage_artifact_dns=false \ + <<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')" + if test "${EXPECTED_REGION}" = us-central1; then + jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \ + <<< "${SOURCE_CELLS}" >/dev/null + fi + CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")" + test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}" + test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \ + "${EXPECTED_UNOBSERVED_BOUND}" + TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")" + MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}" + if test "${DEPLOY_MODE}" = rollback; then + DESIRED_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}" + CURRENT_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}" + DESIRED_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}" + CURRENT_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}" + else + DESIRED_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}" + CURRENT_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}" + DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}" + CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}" + fi + DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}" + OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \ + --arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")" + jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \ + '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-same-cap.tfvars.json" + SERVED_DIGEST="$(gcloud artifacts docker images describe "${DESIRED_IMAGE}" \ + --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" + test "${SERVED_DIGEST}" = "${DESIRED_IMAGE_DIGEST}" + { + echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}" + echo "CELL_ORIGIN=${CELL_ORIGIN}" + echo "TARGET_ZONE=${TARGET_ZONE}" + echo "MIG_NAME=${MIG_NAME}" + echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}" + echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}" + echo "EXPECTED_REGION=${EXPECTED_REGION}" + echo "DESIRED_IMAGE=${DESIRED_IMAGE}" + echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}" + echo "CURRENT_IMAGE_DIGEST=${CURRENT_IMAGE_DIGEST}" + echo "DESIRED_REHOME_PROTOCOL=${DESIRED_REHOME_PROTOCOL}" + echo "CURRENT_REHOME_PROTOCOL=${CURRENT_REHOME_PROTOCOL}" + } >> "${GITHUB_ENV}" + + - name: Verify exact current generation, digest, cap, and rollback point + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + CURRENT_RUNTIME="$(curl --fail-with-body --max-time 30 \ + --request POST "${CELL_ORIGIN}/v1/admin/runtime-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data '{"v":1}')" + # A rollback that failed between template apply and admission restore + # leaves the cell already on the rollback image; resume from that + # state instead of demanding the pre-rollback predecessor. + LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")" + if test "${DEPLOY_MODE}" = rollback \ + && test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then + ROLLBACK_RESUME=true + PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}" + PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}" + else + ROLLBACK_RESUME=false + PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}" + PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}" + fi + RESTORED_MIGRATION_CELLS="$(jq -rn \ + --arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \ + --arg target "${TARGET_CELL_ID}" \ + '$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')" + RESTORED_GENERAL_CELLS="$(jq -rn \ + --arg value "${EXPECTED_GENERAL_CELLS/none/}" \ + --arg target "${TARGET_CELL_ID}" \ + '$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')" + test -n "${RESTORED_MIGRATION_CELLS}" || RESTORED_MIGRATION_CELLS=none + test -n "${RESTORED_GENERAL_CELLS}" || RESTORED_GENERAL_CELLS=none + ISOLATED_MIGRATION_CELLS="$(jq -rn \ + --arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \ + --arg target "${TARGET_CELL_ID}" \ + '$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')" + ISOLATED_GENERAL_CELLS="$(jq -rn \ + --arg value "${EXPECTED_GENERAL_CELLS/none/}" \ + --arg target "${TARGET_CELL_ID}" \ + '$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')" + test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none + test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none + { + echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}" + # The failsafe consumes these; deriving them here keeps them + # defined for a failure in any later step. + echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}" + echo "ISOLATED_GENERAL_CELLS=${ISOLATED_GENERAL_CELLS}" + # No restart happens on resume, so isolate below is skipped and + # cannot advance the selector generation. + echo "SELECTOR_GENERATION_AFTER_ISOLATE=${EFFECTIVE_SELECTOR_GENERATION}" + # A failed-canary rollback enters with the target migration-only, + # so the restore inspect cannot reuse the entry membership inputs. + echo "RESTORED_MIGRATION_CELLS=${RESTORED_MIGRATION_CELLS}" + echo "RESTORED_GENERAL_CELLS=${RESTORED_GENERAL_CELLS}" + } >> "${GITHUB_ENV}" + if ! jq -e --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \ + --arg digest "${PREDECESSOR_IMAGE_DIGEST}" \ + --arg region "${EXPECTED_REGION}" \ + --argjson hardCap "${EXPECTED_HARD_CAP}" \ + --argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \ + --argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \ + --argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \ + && test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \ + '.role == "cell" and .cellId == $cell and .cellUrl == $origin and + (.region == $region or + ($region == "us-central1" and $protocol == 0 and .region == null)) and + .imageDigest == $digest and + .connectionCapacity.hardCap == $hardCap and + .connectionCapacity.unobservedBound == $unobservedBound and + (.draining == false or $drainingOk) and + (.regionalRehomeProtocol // 0) == $protocol' <<< "${CURRENT_RUNTIME}" >/dev/null + then + jq -r --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \ + --arg digest "${PREDECESSOR_IMAGE_DIGEST}" \ + --arg region "${EXPECTED_REGION}" \ + --argjson hardCap "${EXPECTED_HARD_CAP}" \ + --argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \ + --argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \ + --argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \ + && test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \ + '[ + if .role != "cell" then "role" else empty end, + if .cellId != $cell then "cellId" else empty end, + if .cellUrl != $origin then "cellUrl" else empty end, + if (.region != $region and + ($region != "us-central1" or $protocol != 0 or .region != null)) + then "region" else empty end, + if .imageDigest != $digest then "imageDigest" else empty end, + if .connectionCapacity.hardCap != $hardCap then "hardCap" else empty end, + if .connectionCapacity.unobservedBound != $unobservedBound then "unobservedBound" else empty end, + if (.draining != false and ($drainingOk | not)) then "draining" else empty end, + if (.regionalRehomeProtocol // 0) != $protocol then "regionalRehomeProtocol" else empty end + ] | "runtime predecessor mismatch fields=" + join(",")' \ + <<< "${CURRENT_RUNTIME}" >&2 + exit 1 + fi + # The exact legacy digest binds omitted pre-region fields to US and protocol 0. + jq -r '[ + if .region == null then "region" else empty end, + if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end + ] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \ + <<< "${CURRENT_RUNTIME}" + CURRENT_DIRECTOR_STATUS="$(curl --fail-with-body --max-time 30 \ + --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \ + <<< "${CURRENT_DIRECTOR_STATUS}")" + if test "${ROLLBACK_RESUME}" = true && ! jq -e \ + '.status.admissionState == "migration-only"' \ + <<< "${CURRENT_DIRECTOR_STATUS}" >/dev/null; then + echo 'resume requires the isolated migration-only cell a failed rollback leaves' >&2 + exit 1 + fi + [[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]] + echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}" + # Rollback is the documented recovery from a failed canary, which + # leaves the cell migration-only (and possibly still marked + # draining); apply and verify still require a pristine general cell. + if test "${DEPLOY_MODE}" = rollback; then + PRECHECK_ADMISSION=general-or-migration-only + PRECHECK_DRAINING=either + else + PRECHECK_ADMISSION=general + PRECHECK_DRAINING=forbidden + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh --admission "${PRECHECK_ADMISSION}" \ + --draining "${PRECHECK_DRAINING}" --activity allowed \ + --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" + + - name: Finish read-only verification + if: ${{ inputs.mode == 'verify' }} + run: echo 'Exact same-cap rollback point verified.' + + - name: Reversibly isolate and drain only the selected cell + if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} + run: | + echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" + # A cell isolated by a failed canary is already migration-only, so + # isolate is a no-op there that does not advance the selector; the + # result's generation is authoritative either way. + ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --mode isolate)" + echo "${ISOLATE_RESULT}" + ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")" + echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}" + node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --mode drain + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat either --admission migration-only --draining required \ + --activity restart-safe --expected-image-digests "${CURRENT_IMAGE_DIGEST}" \ + --timeout-ms 900000 + + - id: capacity-auth + if: ${{ inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + + - name: Require converged Terraform state and a stable MIG on resume + if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }} + shell: bash + env: + DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} + run: | + # Zero resource changes prove the prior run's apply completed and no + # restart will follow, keeping the incarnation check honest. Root + # outputs may lag a targeted apply, so judge resource_changes only. + terraform -chdir=infra/terraform plan \ + -var-file=environments/production.tfvars \ + -var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \ + -var manage_artifact_dns=false \ + "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ + "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ + -out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan" + if ! terraform -chdir=infra/terraform show -json \ + "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \ + | jq -e '[.resource_changes[]? + | select(.change.actions | any(. != "no-op" and . != "read"))] + | length == 0' >/dev/null + then + # An apply that failed before its template apply also resumes here + # (the cell still serves the rollback image), and repo drift since + # the cell's last roll (for example newly added rehome trust + # config) then legitimately replaces the template. Nothing is + # applied on resume either way, so accept exactly the drift the + # reviewed validator would let a real apply ship for the image the + # cell already serves: the template leaves and re-enters the + # rollback image, as exactly the template-and-MIG change pair. + terraform -chdir=infra/terraform show -json \ + "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \ + | jq -r '"resume found unconverged resources: " + + ([.resource_changes[]? + | select(.change.actions | any(. != "no-op" and . != "read")) + | .address] | join(","))' + echo 'requiring reviewed rollback-image drift' + terraform -chdir=infra/terraform show -json \ + "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --image "${DESIRED_IMAGE}" \ + --rollback-image "${DESIRED_IMAGE}" \ + --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ + --rehome-audience https://relay.onorca.dev/v1/admin/host-drain \ + | jq -e '.changes == 2' >/dev/null + fi + gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 + + - name: Apply only the selected same-cap template and MIG + if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }} + shell: bash + env: + CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} + run: | + terraform -chdir=infra/terraform plan \ + -var-file=environments/production.tfvars \ + -var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \ + -var manage_artifact_dns=false \ + "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ + "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ + -out="${RUNNER_TEMP}/relay-same-cap.tfplan" + terraform -chdir=infra/terraform show -json "${RUNNER_TEMP}/relay-same-cap.tfplan" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \ + --rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \ + --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ + --rehome-audience https://relay.onorca.dev/v1/admin/host-drain + terraform -chdir=infra/terraform apply -auto-approve \ + "${RUNNER_TEMP}/relay-same-cap.tfplan" + gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 + + - id: post-auth + if: ${{ inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Verify new incarnation, exact image, protocol, and durable safety + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh --admission migration-only --draining forbidden \ + --activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \ + --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000 + TARGET_RUNTIME="$(curl --fail-with-body --max-time 30 \ + --request POST "${CELL_ORIGIN}/v1/admin/runtime-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data '{"v":1}')" + jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \ + --argjson protocol "${DESIRED_REHOME_PROTOCOL}" \ + '.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \ + <<< "${TARGET_RUNTIME}" >/dev/null + TARGET_DIRECTOR_STATUS="$(curl --fail-with-body --max-time 30 \ + --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \ + <<< "${TARGET_DIRECTOR_STATUS}")" + if test "${ROLLBACK_RESUME}" = true; then + echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" + # No restart happened; the incarnation legitimately stays put. + test "${TARGET_INCARNATION}" = "${SOURCE_INCARNATION}" + else + test "${TARGET_INCARNATION}" != "${SOURCE_INCARNATION}" + fi + echo "TARGET_INCARNATION=${TARGET_INCARNATION}" >> "${GITHUB_ENV}" + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-selector-generation "${SELECTOR_GENERATION_AFTER_ISOLATE}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \ + --expected-general-cells "${ISOLATED_GENERAL_CELLS}" \ + --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \ + | jq -e '.control.enabled == false' >/dev/null + + - name: Prove exact per-host trust and idempotent no-neighbor behavior + if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} + run: | + node dev/scripts/probe-relay-rehome-trust.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \ + --cell-incarnation "${TARGET_INCARNATION}" + + - name: Restore only the verified selected cell to general admission + if: ${{ inputs.mode != 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} + run: | + echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" + ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --mode activate)" + echo "${ACTIVATE_RESULT}" + SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \ + <<< "${ACTIVATE_RESULT}")" + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh --admission general --draining forbidden --activity allowed \ + --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \ + --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-selector-generation "${SELECTOR_GENERATION_AFTER_ACTIVATE}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \ + --expected-general-cells "${RESTORED_GENERAL_CELLS}" \ + --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \ + | jq -e '.control.enabled == false' >/dev/null + + - id: cleanup-auth + if: ${{ failure() && inputs.mode != 'verify' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Keep a failed cell isolated and rehome disabled + if: ${{ failure() && inputs.mode != 'verify' }} + continue-on-error: true + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }} + run: | + test "${MUTATION_STARTED:-false}" = true || exit 0 + ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --mode isolate)" + echo "${ISOLATE_RESULT}" + # The isolate result carries the authoritative post-isolate generation; + # fixed offsets are wrong whenever an earlier isolate was a no-op. + FAILSAFE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")" + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-selector-generation "${FAILSAFE_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \ + --expected-general-cells "${ISOLATED_GENERAL_CELLS}" \ + --expected-control-generation "${EXPECTED_REHOME_GENERATION}" diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap.yml b/.github/workflows/cloud-deploy-relay-production-same-cap.yml new file mode 100644 index 00000000000..1994966d083 --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-production-same-cap.yml @@ -0,0 +1,310 @@ +name: Deploy Relay Production Same-Cap + +on: + workflow_dispatch: + inputs: + mode: + description: Verify, roll one canary, roll a bounded batch, or roll back + required: true + default: verify + type: choice + options: [verify, canary-apply, batch-apply, rollback] + cell-ids: + description: Ordered comma-separated serving cells; one canary or two to four batch cells + required: true + type: string + target-image-digest: + description: Exact immutable compatibility image digest + required: true + type: string + rollback-image-digest: + description: Exact immutable currently serving rollback digest + required: true + type: string + target-rehome-protocol: + description: Exact target regional-rehome protocol + required: true + default: '1' + type: choice + options: ['0', '1'] + rollback-rehome-protocol: + description: Exact rollback regional-rehome protocol + required: true + default: '0' + type: choice + options: ['0', '1'] + expected-selector-generation: + description: Exact selector generation before the first cell + required: true + type: string + expected-existing-only-cells: + description: Exact existing-only membership, or none + required: true + type: string + expected-migration-only-cells: + description: Exact migration-only membership, or none + required: true + type: string + expected-general-cells: + description: Exact general membership, or none + required: true + type: string + expected-rehome-generation: + description: Exact durable regional-rehome control generation; it must be disabled + required: true + type: string + monitor-run-id: + description: Fresh successful aggregate dry-run monitor workflow run + required: false + type: string + monitor-run-attempt: + description: Exact monitor attempt + required: false + type: string + canary-run-id: + description: Successful same-commit canary run required for batch-apply + required: false + type: string + confirmation: + description: Exact digest-and-cell-bound mutation confirmation + required: false + type: string + +permissions: + actions: read + contents: read + id-token: write + +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + gate: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 10 + environment: production + outputs: + cells: ${{ steps.wave.outputs.cells }} + job-mode: ${{ steps.wave.outputs.job-mode }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: { node-version: 24 } + + - id: wave + env: + MODE: ${{ inputs.mode }} + CELL_IDS: ${{ inputs.cell-ids }} + TARGET_DIGEST: ${{ inputs.target-image-digest }} + ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }} + CONFIRMATION: ${{ inputs.confirmation }} + CANARY_RUN_ID: ${{ inputs.canary-run-id }} + run: | + CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \ + --mode "${MODE}" --cell-ids "${CELL_IDS}" \ + --target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \ + --confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}")" + echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}" + if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then + echo 'job-mode=apply' >> "${GITHUB_OUTPUT}" + else + echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}" + fi + + - name: Download exact prior canary authority + if: ${{ inputs.mode == 'batch-apply' }} + uses: actions/download-artifact@v4 + with: + name: relay-same-cap-canary-${{ inputs.canary-run-id }} + path: ${{ runner.temp }}/relay-same-cap-canary + github-token: ${{ github.token }} + run-id: ${{ inputs.canary-run-id }} + + - name: Verify canary authority against this batch + if: ${{ inputs.mode == 'batch-apply' }} + env: + CANARY_RUN_ID: ${{ inputs.canary-run-id }} + run: | + node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \ + --file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \ + --commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \ + --target-digest "${{ inputs.target-image-digest }}" \ + --rollback-digest "${{ inputs.rollback-image-digest }}" \ + --selector-generation "${{ inputs.expected-selector-generation }}" \ + --rehome-generation "${{ inputs.expected-rehome-generation }}" + + - name: Reject previously consumed aggregate safety evidence + if: ${{ inputs.mode != 'verify' }} + env: + GH_TOKEN: ${{ github.token }} + MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} + MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} + run: | + [[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]] + [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + MARKER_NAME="relay-same-cap-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ + --jq '.total_count')" + test "${COUNT}" = 0 + mkdir -p "${RUNNER_TEMP}/relay-same-cap-monitor-authority" + printf '%s\n' "${GITHUB_RUN_ID}" \ + > "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}" + + - name: Consume aggregate safety evidence for this exact wave + if: ${{ inputs.mode != 'verify' }} + uses: actions/upload-artifact@v4 + with: + name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-same-cap-monitor-authority/relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + retention-days: 90 + if-no-files-found: error + + cell_1: + needs: gate + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + wave-index: '0' + secrets: inherit + + cell_2: + if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }} + needs: [gate, cell_1] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + wave-index: '1' + secrets: inherit + + cell_3: + if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }} + needs: [gate, cell_2] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + wave-index: '2' + secrets: inherit + + cell_4: + if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }} + needs: [gate, cell_3] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + wave-index: '3' + secrets: inherit + + seal_canary: + if: ${{ inputs.mode == 'canary-apply' }} + needs: [gate, cell_1] + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: { node-version: 24 } + + - name: Seal exact successful canary authority + run: | + mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary" + node dev/scripts/relay-production-same-cap-wave.mjs create-canary \ + --cell-id "${{ inputs.cell-ids }}" \ + --target-digest "${{ inputs.target-image-digest }}" \ + --rollback-digest "${{ inputs.rollback-image-digest }}" \ + --confirmation "${{ inputs.confirmation }}" \ + --commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \ + --selector-generation "${{ inputs.expected-selector-generation }}" \ + --rehome-generation "${{ inputs.expected-rehome-generation }}" \ + > "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" + + - uses: actions/upload-artifact@v4 + with: + name: relay-same-cap-canary-${{ github.run_id }} + path: ${{ runner.temp }}/relay-same-cap-canary/authority.json + retention-days: 30 + if-no-files-found: error + + # Every cell job re-enters the run's lease with release: 'false'; only this job frees it. + release_lease: + if: always() + needs: + - gate + - cell_1 + - cell_2 + - cell_3 + - cell_4 + - seal_canary + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 10 + environment: production + steps: + - uses: actions/checkout@v4 + + - uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + release: 'true' diff --git a/.github/workflows/cloud-deploy-relay-production.yml b/.github/workflows/cloud-deploy-relay-production.yml new file mode 100644 index 00000000000..6de990e4c90 --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-production.yml @@ -0,0 +1,219 @@ +name: Deploy Relay Production Candidate + +on: + workflow_dispatch: + inputs: + source-cell-id: + description: Existing Terraform cell ID to evacuate + required: true + type: string + target-cell-id: + description: Distinct Terraform candidate cell ID + required: true + type: string + mode: + description: Audit/preflight are read-only; recover/continue resume committed work; disable/enable/reset/execute mutate admission + required: true + default: preflight + type: choice + options: + - audit + - preflight + - recover-forward + - continue-evacuation + - disable-cell + - enable-empty-cell + - reset-empty-candidate + - execute + confirmation: + description: Enter RECOVER_FORWARD, CONTINUE_EVACUATION, DISABLE_CELL, ENABLE_CELL, RESET_CANDIDATE, or EVACUATE for the matching mutation + required: false + type: string + monitor-run-id: + description: Successful fresh dry-run monitor workflow run ID + required: false + type: string + monitor-run-attempt: + description: Exact dry-run monitor workflow attempt + required: false + type: string + +permissions: + actions: read + contents: read + id-token: write + +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + candidate: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + DIRECTOR_ORIGIN: https://relay.onorca.dev + ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain + SOURCE_CELL_ID: ${{ inputs.source-cell-id }} + TARGET_CELL_ID: ${{ inputs.target-cell-id }} + DEPLOY_MODE: ${{ inputs.mode }} + MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} + MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} + steps: + - uses: actions/checkout@v4 + + - name: Require fresh dry-run evidence reference + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + run: | + [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] + [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + + - name: Download private dry-run evidence + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-monitor-evidence + github-token: ${{ github.token }} + run-id: ${{ inputs.monitor-run-id }} + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: cloud/pnpm-lock.yaml + + - run: pnpm install --frozen-lockfile + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Verify dry-run artifact before cloud authentication + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + run: | + node dev/scripts/relay-monitor-evidence.mjs verify-restore \ + --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run + + - name: Reject previously consumed dry-run evidence + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + COUNT="$(gh api \ + "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ + --jq '.total_count')" + test "${COUNT}" = "0" + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + + - name: Require explicit mutation confirmation + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if [[ "${DEPLOY_MODE}" = "execute" ]]; then + test "${CONFIRMATION}" = "EVACUATE" + elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then + test "${CONFIRMATION}" = "RECOVER_FORWARD" + elif [[ "${DEPLOY_MODE}" = "continue-evacuation" ]]; then + test "${CONFIRMATION}" = "CONTINUE_EVACUATION" + elif [[ "${DEPLOY_MODE}" = "disable-cell" ]]; then + test "${CONFIRMATION}" = "DISABLE_CELL" + elif [[ "${DEPLOY_MODE}" = "enable-empty-cell" ]]; then + test "${CONFIRMATION}" = "ENABLE_CELL" + else + test "${CONFIRMATION}" = "RESET_CANDIDATE" + fi + + - name: Read reviewed Terraform topology + run: | + node dev/scripts/infra.mjs init --env production + terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" + RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)" + echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}" + + - name: Verify fresh dry-run evidence against live selector + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ + --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" \ + --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode dry-run \ + --mutation-mode "${DEPLOY_MODE}" \ + --source-cell-id "${SOURCE_CELL_ID}" \ + --director-origin "${DIRECTOR_ORIGIN}" + + - name: Recheck all live safety signals + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + pnpm incident:relay-preflight -- \ + --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" + + - name: Create single-use dry-run marker + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + run: | + MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" + printf '%s\n' "${GITHUB_RUN_ID}" \ + > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" + + - name: Consume dry-run evidence + if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} + uses: actions/upload-artifact@v4 + with: + name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + retention-days: 90 + if-no-files-found: error + + - name: Preflight or evacuate exact GCE candidate + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/deploy-relay-gce-candidate.mjs \ + --project "${GCP_PROJECT_ID}" \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --admin-audience "${ADMIN_AUDIENCE}" \ + --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \ + --source-cell-id "${SOURCE_CELL_ID}" \ + --target-cell-id "${TARGET_CELL_ID}" \ + --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \ + --mode "${DEPLOY_MODE}" diff --git a/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml b/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml new file mode 100644 index 00000000000..e646980a787 --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml @@ -0,0 +1,109 @@ +name: Deploy Relay Staging GCE Candidate + +on: + workflow_dispatch: + inputs: + source-cell-id: + description: Existing Terraform cell ID to evacuate + required: true + type: string + target-cell-id: + description: Distinct Terraform candidate cell ID + required: true + type: string + mode: + description: Preflight is read-only; reset repairs an empty candidate; execute evacuates + required: true + default: preflight + type: choice + options: + - preflight + - reset-empty-candidate + - execute + confirmation: + description: Enter RESET_CANDIDATE for reset or EVACUATE for execute + required: false + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: relay-staging-mutation + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + candidate: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: staging + env: + GCP_PROJECT_ID: onorca-cloud-staging + DIRECTOR_ORIGIN: https://relay-staging.onorca.dev + ADMIN_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/drain + SOURCE_CELL_ID: ${{ inputs.source-cell-id }} + TARGET_CELL_ID: ${{ inputs.target-cell-id }} + DEPLOY_MODE: ${{ inputs.mode }} + steps: + - uses: actions/checkout@v4 + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Require explicit mutation confirmation + if: ${{ inputs.mode != 'preflight' }} + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if [[ "${DEPLOY_MODE}" = "execute" ]]; then + test "${CONFIRMATION}" = "EVACUATE" + else + test "${CONFIRMATION}" = "RESET_CANDIDATE" + fi + + - name: Read reviewed Terraform topology + run: | + node dev/scripts/infra.mjs init --env staging + terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" + RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)" + echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}" + + - name: Preflight or evacuate exact GCE candidate + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/deploy-relay-gce-candidate.mjs \ + --project "${GCP_PROJECT_ID}" \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --admin-audience "${ADMIN_AUDIENCE}" \ + --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \ + --source-cell-id "${SOURCE_CELL_ID}" \ + --target-cell-id "${TARGET_CELL_ID}" \ + --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \ + --mode "${DEPLOY_MODE}" diff --git a/.github/workflows/cloud-deploy-relay-staging.yml b/.github/workflows/cloud-deploy-relay-staging.yml new file mode 100644 index 00000000000..cdde494f2ca --- /dev/null +++ b/.github/workflows/cloud-deploy-relay-staging.yml @@ -0,0 +1,112 @@ +name: Deploy Relay Staging + +on: + workflow_dispatch: + inputs: + expected-image-digest: + description: Exact checked-in production Relay sha256 digest to deploy + required: true + type: string + +permissions: + contents: read + id-token: write + +concurrency: + # Staging deploy, candidate, auth, and power operations must never overlap. + group: relay-staging-mutation + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + deploy: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: staging + env: + GCP_PROJECT_ID: onorca-cloud-staging + GCP_REGION: ${{ vars.STAGING_GCP_REGION }} + DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging + REPOSITORY_ID: orca-cloud + IMAGE_NAME: relay + EXPECTED_IMAGE_DIGEST: ${{ inputs.expected-image-digest }} + CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + ASIA_PROOF_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }} + REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled + steps: + - uses: actions/checkout@v4 + + - name: Require the expected immutable image + run: '[[ "${EXPECTED_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]' + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.15.8 + terraform_wrapper: false + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Bind the request to the checked-in staging C4 image + shell: bash + run: | + set -euo pipefail + terraform -chdir=infra/terraform init -reconfigure \ + -backend-config=backend/staging.hcl -input=false + IMAGE="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars -var manage_artifact_dns=false \ + <<< 'var.relay_gce_cells["staging-gce-c4"].image' | jq -er '.')" + test "${IMAGE}" = \ + "${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${EXPECTED_IMAGE_DIGEST}" + echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}" + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Require the mirrored immutable image + run: | + DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \ + --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" + test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}" + + - name: Deploy director blue/green + run: | + regional_version="$(gcloud secrets versions describe latest \ + --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \ + --format='value(name)' | awk -F/ '{print $NF}')" + [[ "${regional_version}" =~ ^[1-9][0-9]*$ ]] + RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" + node dev/scripts/deploy-relay-blue-green.mjs \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --service "${DIRECTOR_SERVICE_NAME}" \ + --image "${IMAGE}" \ + --role director \ + --max-instances 2 \ + --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ + --asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \ + --regional-placement-secret-version "${regional_version}" \ + --min-instances 0 \ + --release-id "${RELEASE_ID}" + + - name: Smoke director health + run: | + URL="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(status.url)')" + node dev/scripts/smoke-relay.mjs "${URL}" diff --git a/.github/workflows/cloud-monitor-relay-clock-skew.yml b/.github/workflows/cloud-monitor-relay-clock-skew.yml new file mode 100644 index 00000000000..9d4de846519 --- /dev/null +++ b/.github/workflows/cloud-monitor-relay-clock-skew.yml @@ -0,0 +1,77 @@ +name: Monitor Relay Cell Clock Skew + +# Why: the 2026-08-01 sustained HOST_OFFLINE incident traced to one cell's +# clock running ~100ms ahead, which deterministically failed every host +# challenge under a zero-tolerance freshness check. /health and /ready cannot +# see clock skew; this monitor alarms before drift reaches the (now 2s) +# client tolerance. + +on: + workflow_dispatch: + schedule: + - cron: '17 * * * *' + +permissions: + contents: read + +defaults: + run: + working-directory: cloud + +jobs: + skew: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: ubuntu-latest + steps: + - name: Measure Date-header skew for every relay cell + working-directory: . + run: | + set -u + # Date headers carry whole seconds; compare floored seconds on both + # sides so perfect sync reads 0/±1 and never flaps. An absolute + # floor-skew of >= 2 means real drift of at least ~1s — approaching + # the client's 2s challenge tolerance. Millisecond-precision deltas + # come from the desktop's named-check logging when activations fail. + ALARM_S=2 + failures=0 + reachable=0 + unserved=0 + # Keep this upper bound at or above the highest provisioned cell in + # environments/production.tfvars; unlisted cells are silently unmonitored. + for n in $(seq 1 22); do + cell="c${n}" + url="https://${cell}.relay.onorca.dev/health" + # Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own + # accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as + # perfect sync. Only an HTTP 200 is the relay process itself answering, so only a + # 200 carries a clock worth judging. + response="$(curl -sS -D - -o /dev/null --max-time 8 "${url}" 2>/dev/null || true)" + status="$(printf '%s' "${response}" | awk 'NR==1 {print $2}')" + header="$(printf '%s' "${response}" | tr -d '\r' | grep -i '^date:' || true)" + if [ "${status:-000}" != "200" ] || [ -z "${header}" ]; then + # Expected for the fenced cells; a dead unfenced cell is caught by the heartbeat + # and readiness alerts, not here. Named either way so it is never invisible. + echo "${cell}: not serving (status ${status:-none}); no relay clock to judge" + unserved=$((unserved + 1)) + continue + fi + reachable=$((reachable + 1)) + server_s="$(date -d "${header#*: }" +%s)" + local_s="$(date +%s)" + skew=$((server_s - local_s)) + abs=${skew#-} + if [ "${abs}" -ge "${ALARM_S}" ]; then + echo "::error::${cell}: clock skew ${skew}s reaches ±${ALARM_S}s alarm" + failures=$((failures + 1)) + else + echo "${cell}: skew ${skew}s" + fi + done + echo "cells serving: ${reachable}, not serving: ${unserved}, alarms: ${failures}" + if [ "${reachable}" -eq 0 ]; then + # Now meaningful: previously the load balancer answered for every name, so this + # could never fire and a total fleet outage reported "all healthy". + echo "::error::no relay cell is serving; monitor blind" + exit 1 + fi + exit "$((failures > 0 ? 1 : 0))" diff --git a/.github/workflows/cloud-monitor-relay-production-job.yml b/.github/workflows/cloud-monitor-relay-production-job.yml new file mode 100644 index 00000000000..4e97fb1bb80 --- /dev/null +++ b/.github/workflows/cloud-monitor-relay-production-job.yml @@ -0,0 +1,215 @@ +name: Monitor Relay Production Job + +on: + workflow_call: + inputs: + mode: + required: true + type: string + expected-selector-generation: + required: true + type: string + expected-existing-only-cells: + required: true + type: string + expected-migration-only-cells: + required: true + type: string + expected-general-cells: + required: true + type: string + migration-policy: + required: true + type: string + recovery-source-cell-id: + required: true + type: string + capacity-cell-id: + required: true + type: string + +permissions: + actions: read + contents: read + id-token: write + +defaults: + run: + working-directory: cloud + +jobs: + monitor: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 100 + environment: production + env: + EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} + EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }} + EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }} + EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }} + MIGRATION_POLICY: ${{ inputs.migration-policy }} + RECOVERY_SOURCE_CELL_ID: ${{ inputs.recovery-source-cell-id }} + CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }} + INCIDENT_ID: relay-${{ github.run_id }}-${{ inputs.mode }} + MONITOR_MODE: ${{ inputs.mode }} + OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-incident + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: cloud/pnpm-lock.yaml + + - run: pnpm install --frozen-lockfile + + - id: prior-attempt + if: ${{ github.run_attempt > 1 }} + run: echo "value=$((GITHUB_RUN_ATTEMPT - 1))" >> "${GITHUB_OUTPUT}" + + - name: Restore prior private monitor state + if: ${{ github.run_attempt > 1 }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ steps.prior-attempt.outputs.value }} + path: ${{ github.workspace }}/relay-incident + github-token: ${{ github.token }} + run-id: ${{ github.run_id }} + + - name: Verify restored state provenance + if: ${{ github.run_attempt > 1 }} + run: | + node dev/scripts/relay-monitor-evidence.mjs verify-restore \ + --directory "${OUTPUT_DIRECTORY}" \ + --incident-id "${INCIDENT_ID}" \ + --run-id "${GITHUB_RUN_ID}" \ + --run-attempt "${{ steps.prior-attempt.outputs.value }}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode "${MONITOR_MODE}" + echo "RESTART_FLAG=--restart" >> "${GITHUB_ENV}" + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Verify exact-audience admin identity + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)" + + - name: Run read-only relay dry-run + if: ${{ inputs.mode == 'dry-run' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + pnpm --filter @orca-cloud/relay-ops incident:monitor \ + --environment production \ + --incident-id "${INCIDENT_ID}" \ + --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ + --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ + --migration-policy "${MIGRATION_POLICY}" \ + --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \ + --capacity-cell-id "${CAPACITY_CELL_ID}" \ + --interval-seconds 60 \ + --output-directory "${OUTPUT_DIRECTORY}" \ + --duration-minutes 15 \ + --pre-drain-dry-run \ + ${RESTART_FLAG:-} + + - name: Run first relay monitor segment + if: ${{ inputs.mode == 'monitor' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + pnpm --filter @orca-cloud/relay-ops incident:monitor \ + --environment production \ + --incident-id "${INCIDENT_ID}" \ + --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ + --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ + --migration-policy "${MIGRATION_POLICY}" \ + --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \ + --capacity-cell-id "${CAPACITY_CELL_ID}" \ + --interval-seconds 60 \ + --output-directory "${OUTPUT_DIRECTORY}" \ + --duration-minutes 90 \ + --max-samples-this-run 45 \ + ${RESTART_FLAG:-} + + - id: google-auth-refresh + if: ${{ inputs.mode == 'monitor' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Run remaining relay monitor window + if: ${{ inputs.mode == 'monitor' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-refresh.outputs.id_token }} + run: | + node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)" + pnpm --filter @orca-cloud/relay-ops incident:monitor \ + --environment production \ + --incident-id "${INCIDENT_ID}" \ + --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ + --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ + --migration-policy "${MIGRATION_POLICY}" \ + --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \ + --capacity-cell-id "${CAPACITY_CELL_ID}" \ + --interval-seconds 60 \ + --output-directory "${OUTPUT_DIRECTORY}" \ + --duration-minutes 90 \ + --restart + + - name: Seal private evidence provenance + if: ${{ always() }} + run: | + node dev/scripts/relay-monitor-evidence.mjs create \ + --directory "${OUTPUT_DIRECTORY}" \ + --incident-id "${INCIDENT_ID}" \ + --run-id "${GITHUB_RUN_ID}" \ + --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --mode "${MONITOR_MODE}" + + - name: Publish aggregate job summary + if: ${{ always() }} + run: | + if [[ -f "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" ]]; then + cat "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" >> "${GITHUB_STEP_SUMMARY}" + else + echo "Relay monitor failed before its first aggregate checkpoint." \ + >> "${GITHUB_STEP_SUMMARY}" + fi + + - name: Upload private aggregate evidence + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ github.workspace }}/relay-incident + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/cloud-monitor-relay-production.yml b/.github/workflows/cloud-monitor-relay-production.yml new file mode 100644 index 00000000000..402e2aa9f69 --- /dev/null +++ b/.github/workflows/cloud-monitor-relay-production.yml @@ -0,0 +1,75 @@ +name: Monitor Relay Production + +on: + workflow_dispatch: + inputs: + mode: + description: Run the required 15-minute pre-drain gate or a 90-minute incident watch + required: true + default: dry-run + type: choice + options: + - dry-run + - monitor + expected-selector-generation: + description: Exact durable admission-selector generation + required: true + type: string + expected-existing-only-cells: + description: Exact comma-separated existing-only cells, or none + required: true + type: string + expected-migration-only-cells: + description: Exact comma-separated migration-only cells, or none + required: true + type: string + expected-general-cells: + description: Exact comma-separated general cells, or none + required: true + type: string + migration-policy: + description: Migration checks matched to the intended mutation + required: true + default: strict + type: choice + options: + - strict + - recover-forward + - capacity-transition + recovery-source-cell-id: + description: Existing-only recovery source cell, or none for strict monitoring + required: true + default: none + type: string + capacity-cell-id: + description: General cell for a capacity-transition monitor, or none + required: true + default: none + type: string + +permissions: + actions: read + contents: read + id-token: write + +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + monitor: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + uses: ./.github/workflows/cloud-monitor-relay-production-job.yml + with: + mode: ${{ inputs.mode }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + migration-policy: ${{ inputs.migration-policy }} + recovery-source-cell-id: ${{ inputs.recovery-source-cell-id }} + capacity-cell-id: ${{ inputs.capacity-cell-id }} diff --git a/.github/workflows/cloud-operate-relay-asia-admission.yml b/.github/workflows/cloud-operate-relay-asia-admission.yml new file mode 100644 index 00000000000..7abae2f8646 --- /dev/null +++ b/.github/workflows/cloud-operate-relay-asia-admission.yml @@ -0,0 +1,514 @@ +name: Operate Relay Asia Admission + +on: + workflow_dispatch: + inputs: + environment: + description: Target Relay environment + required: true + type: choice + options: [staging, production] + mode: + description: Inspect, initialize, verify, atomically register, promote, or roll back admission + required: true + default: verify + type: choice + options: [inspect, initialize, verify, register, configure, promote, rollback] + cell-ids: + description: Exact reviewed comma-separated Asia cell wave + required: true + type: string + selector-generation: + description: Exact live selector generation; leave empty only for inspect + required: false + type: string + selector-membership-sha256: + description: Exact fingerprint printed by inspect; required only for initialize + required: false + type: string + selector-attempt-id: + description: Durable unique attempt ID; empty for inspect, verify, and configure + required: false + type: string + image-digest: + description: Expected compatible Relay sha256 digest + required: true + type: string + director-image-digest: + description: Director sha256 digest; required only for configure + required: false + type: string + evidence-run-id: + description: Successful staging or C27 evidence workflow run ID; required for production promotion + required: false + type: string + evidence-run-attempt: + description: Exact evidence workflow run attempt; required for production promotion + required: false + type: string + confirmation: + description: Exact typed confirmation for a mutation + required: false + type: string + +permissions: + actions: read + contents: read + id-token: write + +concurrency: + group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }} + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + admission: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 30 + environment: ${{ inputs.environment }} + env: + DIRECTOR_ORIGIN: ${{ inputs.environment == 'production' && 'https://relay.onorca.dev' || 'https://relay-staging.onorca.dev' }} + AUTH_ORIGIN: ${{ inputs.environment == 'production' && 'https://login.onorca.dev' || 'https://auth-staging.onorca.dev' }} + DIRECTOR_SERVICE: ${{ inputs.environment == 'production' && 'orca-cloud-relay' || 'orca-cloud-relay-staging' }} + REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled + GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }} + GCP_REGION: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_REGION || vars.STAGING_GCP_REGION }} + DIRECTOR_MAX_INSTANCES: ${{ inputs.environment == 'production' && '5' || '2' }} + TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }} + TARGET_ENVIRONMENT: ${{ inputs.environment }} + OPERATION_MODE: ${{ inputs.mode }} + TARGET_CELL_IDS: ${{ inputs.cell-ids }} + EXPECTED_SELECTOR_GENERATION: ${{ inputs.selector-generation }} + EXPECTED_SELECTOR_MEMBERSHIP_SHA256: ${{ inputs.selector-membership-sha256 }} + SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }} + IMAGE_DIGEST: ${{ inputs.image-digest }} + DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }} + EVIDENCE_RUN_ID: ${{ inputs.evidence-run-id }} + EVIDENCE_RUN_ATTEMPT: ${{ inputs.evidence-run-attempt }} + OPERATION_CONFIRMATION: ${{ inputs.confirmation }} + DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + DEPLOY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + steps: + - uses: actions/checkout@v4 + + - name: Validate exact operation inputs before authentication + id: inputs + shell: bash + run: | + set -euo pipefail + test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}" + test -n "${DEPLOY_SERVICE_ACCOUNT}" + [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] + evidence_kind=none + if test "${OPERATION_MODE}" = inspect; then + test -z "${EXPECTED_SELECTOR_GENERATION}" + test -z "${SELECTOR_ATTEMPT_ID}" + test -z "${OPERATION_CONFIRMATION}" + test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" + test -z "${DIRECTOR_IMAGE_DIGEST}" + else + [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] + fi + if test "${OPERATION_MODE}" = initialize; then + test "${EXPECTED_SELECTOR_GENERATION}" = 0 + [[ "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" =~ ^[a-f0-9]{64}$ ]] + test -z "${DIRECTOR_IMAGE_DIGEST}" + [[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] + test "${OPERATION_CONFIRMATION}" = INITIALIZE_ADMISSION_SELECTOR + elif test "${OPERATION_MODE}" = verify; then + test -z "${SELECTOR_ATTEMPT_ID}" + test -z "${OPERATION_CONFIRMATION}" + test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" + test -z "${DIRECTOR_IMAGE_DIGEST}" + elif test "${OPERATION_MODE}" = inspect; then + : + elif test "${OPERATION_MODE}" = configure; then + test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" + test -z "${SELECTOR_ATTEMPT_ID}" + [[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] + test "${OPERATION_CONFIRMATION}" = CONFIGURE_ASIA_DIRECTOR + else + test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" + test -z "${DIRECTOR_IMAGE_DIGEST}" + [[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] + case "${OPERATION_MODE}:${OPERATION_CONFIRMATION}" in + register:REGISTER_ASIA_MIGRATION_ONLY) ;; + promote:PROMOTE_ASIA_GENERAL) ;; + rollback:ROLLBACK_ASIA_MIGRATION_ONLY) ;; + *) echo "typed confirmation does not match the requested mutation" >&2; exit 1 ;; + esac + fi + if test "${TARGET_ENVIRONMENT}:${OPERATION_MODE}" = production:promote; then + [[ "${EVIDENCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]] + [[ "${EVIDENCE_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + case "${TARGET_CELL_IDS}" in + production-gce-c27) + test "${#SELECTOR_ATTEMPT_ID}" -le 119 + evidence_kind=staging + artifact_name="relay-asia-staging-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}" + ;; + production-gce-c28,production-gce-c29) + evidence_kind=c27 + artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}" + ;; + *) echo "production promotion wave is not reviewed" >&2; exit 1 ;; + esac + else + test -z "${EVIDENCE_RUN_ID}" + test -z "${EVIDENCE_RUN_ATTEMPT}" + artifact_name=none + fi + { + echo "evidence_kind=${evidence_kind}" + echo "artifact_name=${artifact_name}" + } >> "${GITHUB_OUTPUT}" + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }} + + - name: Install exact C27 canary dependencies + if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }} + run: pnpm install --frozen-lockfile + + - name: Build the C27 canary Relay contract + if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }} + run: pnpm --filter @orca-cloud/relay-contract build + + - name: Download immutable rollout evidence + if: ${{ steps.inputs.outputs.evidence_kind != 'none' }} + uses: actions/download-artifact@v4 + with: + name: ${{ steps.inputs.outputs.artifact_name }} + path: ${{ runner.temp }}/relay-asia-input-evidence + github-token: ${{ github.token }} + run-id: ${{ inputs.evidence-run-id }} + + - name: Verify evidence provenance and rollout binding before authentication + if: ${{ steps.inputs.outputs.evidence_kind != 'none' }} + env: + GH_TOKEN: ${{ github.token }} + EVIDENCE_KIND: ${{ steps.inputs.outputs.evidence_kind }} + shell: bash + run: | + set -euo pipefail + run_json="${RUNNER_TEMP}/relay-asia-evidence-run.json" + verified="${RUNNER_TEMP}/relay-asia-evidence-verified" + gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${EVIDENCE_RUN_ID}/attempts/${EVIDENCE_RUN_ATTEMPT}" > "${run_json}" + evidence_commit_sha="$( + jq -er '.head_sha | select(type == "string" and test("^[a-f0-9]{40}$"))' "${run_json}" + )" + command=(node dev/scripts/relay-asia-rollout-evidence.mjs "verify-${EVIDENCE_KIND}" + --evidence "${RUNNER_TEMP}/relay-asia-input-evidence/evidence.json" + --run-json "${run_json}" + --commit-sha "${evidence_commit_sha}" + --image-digest "${IMAGE_DIGEST}" + --now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + --output "${verified}") + if test "${EVIDENCE_KIND}" = c27; then + command+=(--selector-generation "${EXPECTED_SELECTOR_GENERATION}") + fi + "${command[@]}" + test "$(< "${verified}")" = verified + + - id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ env.DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: ${{ env.DIRECTOR_ORIGIN }}/v1/admin/drain + id_token_include_email: true + + - name: Require the exact director image before promotion + if: ${{ inputs.mode == 'promote' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + runtime="$(curl --fail-with-body --max-time 30 --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data '{"v":1}')" + test "$(jq -r '.role' <<< "${runtime}")" = director + test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}" + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }} + object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }} + if: ${{ inputs.mode == 'configure' || (inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27') }} + + - uses: hashicorp/setup-terraform@v3 + if: ${{ inputs.mode == 'configure' }} + with: + terraform_version: 1.15.8 + terraform_wrapper: false + + - name: Run the exact generation-bound admission operation + id: admission-operation + if: ${{ inputs.mode != 'configure' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + run: | + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment "${TARGET_ENVIRONMENT}" \ + --mode "${OPERATION_MODE}" \ + --cell-ids "${TARGET_CELL_IDS}" \ + --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-membership-sha256 "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" \ + --attempt-id "${SELECTOR_ATTEMPT_ID}" \ + --image-digest "${IMAGE_DIGEST}")" + generation="$(jq -er '.generation' <<< "${result}")" + states="$(jq -cS '.states // {}' <<< "${result}")" + membership="$(jq -cS '.membership // empty' <<< "${result}")" + membership_sha256="$(jq -r '.membershipSha256 // empty' <<< "${result}")" + echo "generation=${generation}" >> "${GITHUB_OUTPUT}" + result_dir="${RUNNER_TEMP}/relay-asia-admission-result" + mkdir -p "${result_dir}" + node dev/scripts/sanitize-relay-asia-admission-result.mjs \ + <<< "${result}" > "${result_dir}/result.json" + { + echo "### Relay Asia admission" + echo "- Mode: ${OPERATION_MODE}" + echo "- Cells: ${TARGET_CELL_IDS}" + echo "- Result generation: ${generation}" + echo "- States: \`${states}\`" + if test -n "${membership}"; then echo "- Membership: \`${membership}\`"; fi + if test -n "${membership_sha256}"; then + echo "- Membership SHA-256: \`${membership_sha256}\`" + fi + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Verify C27 state and start the timed canary + id: c27-start + if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment production \ + --mode verify \ + --cell-ids production-gce-c27,production-gce-c28,production-gce-c29 \ + --expected-generation "${{ steps.admission-operation.outputs.generation }}" \ + --image-digest "${IMAGE_DIGEST}")" + test "$(jq -r '.states["production-gce-c27"]' <<< "${result}")" = general + test "$(jq -r '.states["production-gce-c28"]' <<< "${result}")" = migration-only + test "$(jq -r '.states["production-gce-c29"]' <<< "${result}")" = migration-only + echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}" + + - name: Run a real five-minute C27 control and splice canary + id: c27-load + if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }} + shell: bash + run: | + set -euo pipefail + log="${RUNNER_TEMP}/relay-asia-c27-load.jsonl" + report="${RUNNER_TEMP}/relay-asia-c27-load.json" + node dev/scripts/load-relay-controls.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --auth-origin "${AUTH_ORIGIN}" \ + --preferred-region asia-east2 \ + --relay-asia-load-principals 1 \ + --controls 1 \ + --splices 1 \ + --capacity-hard-cap 3000 \ + --ramp-seconds 0 \ + --duration-seconds 300 \ + --splice-hold-seconds 60 \ + --required-lease-horizons 2 > "${log}" + jq -cer 'select(.event == "relay_load_complete")' "${log}" | tail -n 1 > "${report}" + echo "ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}" + + - name: Collect regional, Relay SQL, and Cloud SQL canary evidence + if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + CANARY_STARTED_AT: ${{ steps.c27-start.outputs.started_at }} + shell: bash + run: | + set -euo pipefail + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment production \ + --mode verify \ + --cell-ids production-gce-c27,production-gce-c28,production-gce-c29 \ + --expected-generation "${{ steps.admission-operation.outputs.generation }}" \ + --image-digest "${IMAGE_DIGEST}")" + test "$(jq -r '.states["production-gce-c27"]' <<< "${result}")" = general + ended_at="${{ steps.c27-load.outputs.ended_at }}" + sleep 60 + output="${RUNNER_TEMP}/relay-asia-output-evidence" + logs="${RUNNER_TEMP}/relay-asia-c27-runtime-metrics.json" + mkdir -p "${output}" + gcloud logging read \ + "timestamp>=\"${CANARY_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \ + --project "${GCP_PROJECT_ID}" \ + --limit 20000 \ + --format json > "${logs}" + node dev/scripts/relay-asia-rollout-evidence.mjs create-c27 \ + --repository "${GITHUB_REPOSITORY}" \ + --run-id "${GITHUB_RUN_ID}" \ + --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --image-digest "${IMAGE_DIGEST}" \ + --selector-generation "${{ steps.admission-operation.outputs.generation }}" \ + --started-at "${CANARY_STARTED_AT}" \ + --ended-at "${ended_at}" \ + --load-report "${RUNNER_TEMP}/relay-asia-c27-load.json" \ + --logs-json "${logs}" \ + --output "${output}/evidence.json" + jq -r '.metrics | to_entries[] | "- \(.key): \(.value)"' \ + "${output}/evidence.json" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload immutable C27 canary evidence + id: c27-evidence-upload + if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }} + uses: actions/upload-artifact@v4 + with: + name: relay-asia-c27-canary-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/relay-asia-output-evidence/evidence.json + if-no-files-found: error + retention-days: 7 + + - name: Upload sanitized admission result + if: ${{ inputs.mode != 'configure' && steps.admission-operation.outcome == 'success' }} + uses: actions/upload-artifact@v4 + with: + name: relay-asia-admission-result-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/relay-asia-admission-result/result.json + if-no-files-found: error + retention-days: 7 + + - name: Return an unproven C27 canary to migration-only + if: ${{ always() && inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' && steps.admission-operation.outcome != 'skipped' && steps.c27-evidence-upload.outcome != 'success' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment production \ + --mode recover-promotion \ + --cell-ids production-gce-c27 \ + --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --attempt-id "${SELECTOR_ATTEMPT_ID}" \ + --image-digest "${IMAGE_DIGEST}")" + if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi + promoted_generation="$(jq -er '.generation' <<< "${promoted}")" + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment production \ + --mode rollback \ + --cell-ids production-gce-c27 \ + --expected-generation "${promoted_generation}" \ + --attempt-id "${SELECTOR_ATTEMPT_ID}-rollback" \ + --image-digest "${IMAGE_DIGEST}")" + test "$(jq -r '.states["production-gce-c27"]' <<< "${result}")" = migration-only + + - name: Require registered migration-only cells before director configuration + if: ${{ inputs.mode == 'configure' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + run: | + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment "${TARGET_ENVIRONMENT}" \ + --mode registered \ + --cell-ids "${TARGET_CELL_IDS}" \ + --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --image-digest "${IMAGE_DIGEST}")" + test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true + + - name: Build the additive director cell configuration + if: ${{ inputs.mode == 'configure' }} + id: director-config + shell: bash + run: | + set -euo pipefail + terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}" + topology="${RUNNER_TEMP}/relay-asia-state-topology.json" + current="${RUNNER_TEMP}/relay-current-director-cells.json" + desired="${RUNNER_TEMP}/relay-asia-director-cells.json" + terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${topology}" + service="$(gcloud run services describe "${DIRECTOR_SERVICE}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" + revision="$(jq -er '[.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end' \ + <<< "${service}")" + gcloud run revisions describe "${revision}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -er '.spec.containers[0].env[] | select(.name == "ORCA_RELAY_CELLS_JSON") | .value | fromjson' \ + > "${current}" + node dev/scripts/prepare-relay-asia-director-cells.mjs \ + --current-json "${current}" \ + --topology-json "${topology}" \ + --output "${desired}" \ + --cell-ids "${TARGET_CELL_IDS}" \ + --image-digest "${IMAGE_DIGEST}" + echo "file=${desired}" >> "${GITHUB_OUTPUT}" + + - name: Deploy the registered additive director topology + if: ${{ inputs.mode == 'configure' }} + env: + DIRECTOR_CELLS_FILE: ${{ steps.director-config.outputs.file }} + run: | + current="$(gcloud secrets versions access latest \ + --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")" + [[ "${current}" =~ ^(true|false)$ ]] + image="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" + release_id="asia-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" + node dev/scripts/deploy-relay-blue-green.mjs \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --service "${DIRECTOR_SERVICE}" \ + --image "${image}" \ + --role director \ + --max-instances "${DIRECTOR_MAX_INSTANCES}" \ + --release-id "${release_id}" \ + --director-cells-json "$(< "${DIRECTOR_CELLS_FILE}")" \ + --prune-revisions false + + - name: Verify selector and heartbeats after director configuration + if: ${{ inputs.mode == 'configure' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + run: | + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment "${TARGET_ENVIRONMENT}" \ + --mode verify \ + --cell-ids "${TARGET_CELL_IDS}" \ + --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --image-digest "${IMAGE_DIGEST}")" + test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true + revision="$(gcloud run services describe "${DIRECTOR_SERVICE}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -er '[.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end')" + revision_json="$(gcloud run revisions describe "${revision}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" + jq -e --arg image "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" \ + '.spec.containers[0].image == $image' <<< "${revision_json}" > /dev/null + jq -er --arg secret "${REGIONAL_PLACEMENT_SECRET}" \ + '[.spec.containers[0].env[] | + select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") | + (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) | + {secret: (.secret // .name), version: (.version // .key)} | + select(.secret == $secret and (.version | test("^[1-9][0-9]*$")))] | + if length == 1 then .[0].version else error("regional switch version missing") end' \ + <<< "${revision_json}" > "${RUNNER_TEMP}/relay-regional-placement-version" + regional_version="$(< "${RUNNER_TEMP}/relay-regional-placement-version")" + [[ "$(gcloud secrets versions access "${regional_version}" --project "${GCP_PROJECT_ID}" \ + --secret "${REGIONAL_PLACEMENT_SECRET}")" =~ ^(true|false)$ ]] + echo "Director configuration now lists the registered migration-only Asia cells." >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-operate-relay-production-rehome-job.yml b/.github/workflows/cloud-operate-relay-production-rehome-job.yml new file mode 100644 index 00000000000..1ceadcece12 --- /dev/null +++ b/.github/workflows/cloud-operate-relay-production-rehome-job.yml @@ -0,0 +1,328 @@ +name: Operate Relay Production Rehome Job + +on: + workflow_call: + inputs: + mode: { required: true, type: string } + director-image-digest: { required: true, type: string } + rollback-image-digest: { required: true, type: string } + expected-selector-generation: { required: true, type: string } + expected-existing-only-cells: { required: true, type: string } + expected-migration-only-cells: { required: true, type: string } + expected-general-cells: { required: true, type: string } + expected-control-generation: { required: true, type: string } + not-before: { required: true, type: string } + rate-per-minute: { required: true, type: string } + preference-max-age-ms: { required: true, type: string } + drain-grace-ms: { required: true, type: string } + confirmation: { required: true, type: string } + monitor-run-id: { required: true, type: string } + monitor-run-attempt: { required: true, type: string } + +permissions: + actions: read + contents: read + id-token: write + +defaults: + run: + working-directory: cloud + +jobs: + control: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 30 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} + DIRECTOR_SERVICE: orca-cloud-relay + DIRECTOR_ORIGIN: https://relay.onorca.dev + REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain + MODE: ${{ inputs.mode }} + DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }} + ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }} + EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} + EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }} + EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }} + EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }} + EXPECTED_CONTROL_GENERATION: ${{ inputs.expected-control-generation }} + NOT_BEFORE: ${{ inputs.not-before }} + RATE_PER_MINUTE: ${{ inputs.rate-per-minute }} + PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }} + DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }} + CONFIRMATION: ${{ inputs.confirmation }} + MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} + MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} + OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence + steps: + - name: Require exact reusable-workflow configuration + working-directory: . + env: + DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} + run: | + [[ "${MODE}" =~ ^(inspect|enable|pause|disable)$ ]] + [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] + [[ "${EXPECTED_CONTROL_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] + test -n "${DEPLOY_WIF}" + test -n "${DEPLOY_SERVICE_ACCOUNT}" + if [[ "${MODE}" =~ ^(inspect|enable)$ ]]; then + [[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + [[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + test -n "${GCP_REGION}" + test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" + fi + case "${MODE}" in + inspect) + test -z "${CONFIRMATION}" + ;; + enable) + test "${CONFIRMATION}" = ENABLE_REGIONAL_REHOMING + test "${RATE_PER_MINUTE}" = 10 + [[ "${NOT_BEFORE}" =~ ^[1-9][0-9]*$ ]] + ;; + pause) + test "${CONFIRMATION}" = PAUSE_REGIONAL_REHOMING + ;; + disable) + test "${CONFIRMATION}" = DISABLE_REGIONAL_REHOMING + ;; + esac + + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Apply emergency durable pause or disable before diagnostics + if: ${{ inputs.mode == 'pause' || inputs.mode == 'disable' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ + --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ + --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ + --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \ + --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \ + --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \ + | tee "${RUNNER_TEMP}/relay-rehome-control.json" + + - uses: pnpm/action-setup@v4 + if: ${{ inputs.mode == 'enable' }} + with: { package_json_file: cloud/package.json } + + - run: pnpm install --frozen-lockfile + if: ${{ inputs.mode == 'enable' }} + + - name: Download fresh aggregate safety evidence + if: ${{ inputs.mode == 'enable' }} + uses: actions/download-artifact@v4 + with: + name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ github.workspace }}/relay-monitor-evidence + github-token: ${{ github.token }} + run-id: ${{ inputs.monitor-run-id }} + + - name: Verify enable evidence provenance + if: ${{ inputs.mode == 'enable' }} + run: | + [[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]] + [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + node dev/scripts/relay-monitor-evidence.mjs verify-authority \ + --directory "${OUTPUT_DIRECTORY}" \ + --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ + --run-id "${MONITOR_RUN_ID}" --run-attempt "${MONITOR_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" --mode dry-run \ + --required-migration-policy strict + + - name: Reject previously consumed enable safety evidence + if: ${{ inputs.mode == 'enable' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ + --jq '.total_count')" + test "${COUNT}" = 0 + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-terraform-state + object: terraform/state/cloud-sql-rollout/production.lock + + - name: Verify exact serving and rollback director identities + if: ${{ inputs.mode == 'inspect' || inputs.mode == 'enable' }} + env: + DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} + run: | + SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" + SERVING_REVISION="$(jq -er \ + '[.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName + else error("director does not have one serving revision") end' \ + <<< "${SERVICE_JSON}")" + ROLLBACK_REVISION="$(jq -er \ + '[.status.traffic[] | select(.tag == "selector-rollback")] | + if length == 1 then .[0].revisionName else error("rollback tag missing") end' \ + <<< "${SERVICE_JSON}")" + verify_revision() { + local revision="$1" expected_digest="$2" + local json + json="$(gcloud run revisions describe "${revision}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" + test "$(jq -r '.spec.serviceAccountName' <<< "${json}")" = \ + "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" + test "$(jq -r '.spec.containers[0].image | split("@") | last' <<< "${json}")" = \ + "${expected_digest}" + test "$(jq -r '[.spec.containers[0].env[] | select(.name == + "ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT") | .value] | if length == 1 + then .[0] else empty end' <<< "${json}")" = "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" + test "$(jq -r '[.spec.containers[0].env[] | select(.name == + "ORCA_RELAY_REHOME_AUDIENCE") | .value] | if length == 1 then .[0] + else empty end' <<< "${json}")" = "${REHOME_AUDIENCE}" + } + verify_revision "${SERVING_REVISION}" "${DIRECTOR_IMAGE_DIGEST}" + verify_revision "${ROLLBACK_REVISION}" "${ROLLBACK_IMAGE_DIGEST}" + + - name: Seal 24-hour aggregate region observation evidence + if: ${{ inputs.mode == 'enable' }} + run: | + mkdir -p "${RUNNER_TEMP}/relay-region-observation" + # 6 director instances x 120 samples/hour x 25h = 18000; a clipped + # read empties the oldest hourly buckets and fails the seal. + gcloud logging read \ + 'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND jsonPayload.event="orca_relay_runtime_metrics" AND jsonPayload.role="director"' \ + --project "${GCP_PROJECT_ID}" --freshness=25h --limit=30000 --format=json \ + | node dev/scripts/relay-region-observation-evidence.mjs create \ + --commit-sha "${GITHUB_SHA}" \ + --director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \ + --selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --control-generation "${EXPECTED_CONTROL_GENERATION}" \ + > "${RUNNER_TEMP}/relay-region-observation/evidence.json" + + - name: Upload sealed 24-hour aggregate region evidence + if: ${{ inputs.mode == 'enable' }} + uses: actions/upload-artifact@v4 + with: + name: relay-region-observation-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/relay-region-observation/evidence.json + retention-days: 90 + if-no-files-found: error + + - name: Verify sealed 24-hour enable authority + if: ${{ inputs.mode == 'enable' }} + run: | + node dev/scripts/relay-region-observation-evidence.mjs verify \ + --file "${RUNNER_TEMP}/relay-region-observation/evidence.json" \ + --commit-sha "${GITHUB_SHA}" \ + --director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \ + --selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --control-generation "${EXPECTED_CONTROL_GENERATION}" + + - name: Recheck every aggregate safety signal before enable + if: ${{ inputs.mode == 'enable' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + pnpm incident:relay-preflight -- \ + --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" + + - name: Seal single-use enable safety authority + if: ${{ inputs.mode == 'enable' }} + run: | + MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" + mkdir -p "${RUNNER_TEMP}/relay-rehome-enable-authority" + printf '%s\n' "${GITHUB_RUN_ID}" \ + > "${RUNNER_TEMP}/relay-rehome-enable-authority/${MARKER_NAME}" + + - name: Consume enable safety evidence before durable mutation + if: ${{ inputs.mode == 'enable' }} + uses: actions/upload-artifact@v4 + with: + name: relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + path: ${{ runner.temp }}/relay-rehome-enable-authority/relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} + retention-days: 90 + if-no-files-found: error + + - name: Inspect regional rehome control + if: ${{ inputs.mode == 'inspect' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ + --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ + --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ + | tee "${RUNNER_TEMP}/relay-rehome-control.json" + + - name: Apply exact durable regional rehome enable + if: ${{ inputs.mode == 'enable' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ + --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ + --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ + --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ + --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \ + --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \ + --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \ + | tee "${RUNNER_TEMP}/relay-rehome-control.json" + + - name: Read fresh aggregate completion and abort evidence + run: | + gcloud logging read \ + 'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND textPayload:"[orca-relay] regional rehome inventory"' \ + --project "${GCP_PROJECT_ID}" --freshness=15m --limit=20 --format=json \ + | node dev/scripts/relay-rehome-aggregate-evidence.mjs --max-age-ms 900000 \ + | tee "${RUNNER_TEMP}/relay-rehome-inventory.json" + + - name: Publish aggregate control evidence + run: | + { + echo '### Regional rehome control' + jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \ + "${RUNNER_TEMP}/relay-rehome-control.json" + jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`"' \ + "${RUNNER_TEMP}/relay-rehome-inventory.json" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Fail closed after an unsuccessful enable run + if: ${{ failure() && inputs.mode == 'enable' && steps.google-auth.outcome == 'success' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/operate-relay-regional-rehome.mjs \ + --mode recover-enable --director-origin "${DIRECTOR_ORIGIN}" \ + --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ + --confirmation RECOVER_FAILED_REGIONAL_REHOME_ENABLE \ + | tee "${RUNNER_TEMP}/relay-rehome-enable-recovery.json" + jq -e \ + '.mode == "recover-enable" and .control.enabled == false' \ + "${RUNNER_TEMP}/relay-rehome-enable-recovery.json" >/dev/null diff --git a/.github/workflows/cloud-operate-relay-production-rehome.yml b/.github/workflows/cloud-operate-relay-production-rehome.yml new file mode 100644 index 00000000000..40bf5ebbd4f --- /dev/null +++ b/.github/workflows/cloud-operate-relay-production-rehome.yml @@ -0,0 +1,106 @@ +name: Operate Relay Production Rehome + +on: + workflow_dispatch: + inputs: + mode: + description: Inspect or apply the durable regional-rehome switch + required: true + default: inspect + type: choice + options: [inspect, enable, pause, disable] + director-image-digest: + description: Exact immutable serving director digest + required: true + type: string + rollback-image-digest: + description: Exact immutable selector-rollback director digest + required: true + type: string + expected-selector-generation: + description: Exact admission selector generation + required: true + type: string + expected-existing-only-cells: + description: Exact existing-only membership, or none + required: true + type: string + expected-migration-only-cells: + description: Exact migration-only membership, or none + required: true + type: string + expected-general-cells: + description: Exact general membership, or none + required: true + type: string + expected-control-generation: + description: Exact durable rehome generation + required: true + type: string + not-before: + description: Exact epoch milliseconds; ignored only by inspect + required: true + default: '0' + type: string + rate-per-minute: + description: Exact global host rate; initial enable is fixed at 10 + required: true + default: '10' + type: string + preference-max-age-ms: + description: Maximum fresh preference age + required: true + default: '86400000' + type: string + drain-grace-ms: + description: Per-host source drain grace + required: true + default: '3600000' + type: string + monitor-run-id: + description: Fresh successful aggregate dry-run required only by enable + required: false + type: string + monitor-run-attempt: + description: Exact monitor attempt required only by enable + required: false + type: string + confirmation: + description: ENABLE_REGIONAL_REHOMING, PAUSE_REGIONAL_REHOMING, or DISABLE_REGIONAL_REHOMING + required: false + type: string + +permissions: + actions: read + contents: read + id-token: write + +concurrency: + group: production-cloud-sql-rollout + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + operate: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + uses: ./.github/workflows/cloud-operate-relay-production-rehome-job.yml + with: + mode: ${{ inputs.mode }} + director-image-digest: ${{ inputs.director-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-control-generation: ${{ inputs.expected-control-generation }} + not-before: ${{ inputs.not-before }} + rate-per-minute: ${{ inputs.rate-per-minute }} + preference-max-age-ms: ${{ inputs.preference-max-age-ms }} + drain-grace-ms: ${{ inputs.drain-grace-ms }} + confirmation: ${{ inputs.confirmation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + secrets: inherit diff --git a/.github/workflows/cloud-power-relay-staging.yml b/.github/workflows/cloud-power-relay-staging.yml new file mode 100644 index 00000000000..0e8311e4ec2 --- /dev/null +++ b/.github/workflows/cloud-power-relay-staging.yml @@ -0,0 +1,101 @@ +name: Power Relay Staging + +on: + schedule: + # A zero-activity guard makes this a no-op when an internal test is still running. + - cron: '0 9 * * *' + workflow_dispatch: + inputs: + mode: + description: Inspect, wake, or sleep the staging Relay data plane + required: true + default: status + type: choice + options: + - status + - wake + - sleep + wake-cells: + description: Wake configured admission cells, or include disabled candidate cells + required: true + default: configured + type: choice + options: + - configured + - all + confirmation: + description: Enter WAKE_STAGING or SLEEP_STAGING for a manual mutation + required: false + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: relay-staging-mutation + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + power: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: staging + env: + POWER_MODE: ${{ github.event_name == 'schedule' && 'sleep' || inputs.mode }} + WAKE_CELLS: ${{ inputs.wake-cells || 'configured' }} + steps: + - uses: actions/checkout@v4 + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Require explicit manual mutation confirmation + if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode != 'status' }} + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if [[ "${POWER_MODE}" = "wake" ]]; then + test "${CONFIRMATION}" = "WAKE_STAGING" + else + test "${CONFIRMATION}" = "SLEEP_STAGING" + fi + + - name: Read reviewed staging topology + run: | + node dev/scripts/infra.mjs init --env staging + terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" + + - name: Inspect or change staging power state + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/power-staging-relay.mjs \ + --mode "${POWER_MODE}" \ + --wake-cells "${WAKE_CELLS}" \ + --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" diff --git a/.github/workflows/cloud-prove-relay-asia-staging.yml b/.github/workflows/cloud-prove-relay-asia-staging.yml new file mode 100644 index 00000000000..9a66e967b57 --- /dev/null +++ b/.github/workflows/cloud-prove-relay-asia-staging.yml @@ -0,0 +1,331 @@ +name: Prove Relay Asia Staging + +on: + workflow_dispatch: + inputs: + image-digest: + description: Exact immutable Relay digest deployed on staging C4 + required: true + type: string + selector-generation: + description: Exact selector generation with C4 migration-only + required: true + type: string + promote-attempt-id: + description: Durable unique C4 promotion attempt ID + required: true + type: string + rollback-attempt-id: + description: Durable unique C4 rollback attempt ID + required: true + type: string + confirmation: + description: Enter PROVE_ASIA_STAGING + required: true + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: relay-staging-mutation + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + prove: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} + runs-on: [self-hosted, linux, x64, relay-asia-east2-load] + timeout-minutes: 75 + environment: staging + env: + GCP_PROJECT_ID: onorca-cloud-staging + DIRECTOR_ORIGIN: https://relay-staging.onorca.dev + AUTH_ORIGIN: https://auth-staging.onorca.dev + IMAGE_DIGEST: ${{ inputs.image-digest }} + INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }} + PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }} + ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }} + steps: + - uses: actions/checkout@v4 + + - name: Validate the exact staging proof request + shell: bash + run: | + set -euo pipefail + test "${{ inputs.confirmation }}" = PROVE_ASIA_STAGING + [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]] + [[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] + [[ "${ROLLBACK_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] + test "${PROMOTE_ATTEMPT_ID}" != "${ROLLBACK_ATTEMPT_ID}" + + - id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - name: Require the exact staging director image before promotion + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + runtime="$(curl --fail-with-body --max-time 30 --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data '{"v":1}')" + test "$(jq -r '.role' <<< "${runtime}")" = director + test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}" + + - name: Install exact load-harness dependencies + run: pnpm install --frozen-lockfile + + - name: Build the Relay load-harness contract + run: pnpm --filter @orca-cloud/relay-contract build + + - name: Promote only staging C4 + id: promote + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging \ + --mode promote \ + --cell-ids staging-gce-c4 \ + --expected-generation "${INITIAL_SELECTOR_GENERATION}" \ + --attempt-id "${PROMOTE_ATTEMPT_ID}" \ + --image-digest "${IMAGE_DIGEST}")" + generation="$(jq -er '.generation' <<< "${result}")" + test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = general + echo "generation=${generation}" >> "${GITHUB_OUTPUT}" + echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}" + + - name: Run sharded two-horizon and mixed splice proofs + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof" + mkdir -p "${proof_dir}" + fd_limit="$(ulimit -n)" + if test "${fd_limit}" != unlimited; then + [[ "${fd_limit}" =~ ^[0-9]+$ ]] + test "${fd_limit}" -ge 4096 + fi + run_phase() { + phase="$1" + controls="$2" + splices="$3" + pids=() + stop_shards() { + for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done + for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done + } + trap stop_shards EXIT + for shard in 0 1 2 3; do + slow=0 + wedged=0 + boundary_args=() + request_unit_args=() + if test "${phase}" = launch && test "${shard}" = 0; then + slow=4 + wedged=1 + fi + if test "${phase}" = launch && test "${shard}" = 0; then + boundary_args=( + --region-behavior-probes 1 + --capacity-cell-id staging-gce-c4 + --capacity-cell-origin https://c4.relay-staging.onorca.dev + --capacity-unobserved-bound 60 + --rebind-probes 2 + --skip-rebind-overflow-check + ) + fi + node dev/scripts/load-relay-controls.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --auth-origin "${AUTH_ORIGIN}" \ + --preferred-region asia-east2 \ + --relay-asia-load-principals 32 \ + --controls "${controls}" \ + --splices "${splices}" \ + --slow-reader-splices "${slow}" \ + --wedged-reader-splices "${wedged}" \ + --capacity-hard-cap 3000 \ + "${boundary_args[@]}" \ + "${request_unit_args[@]}" \ + --phase-barrier-dir "${proof_dir}/${phase}-barrier" \ + --aggregate-controls "$((controls * 4))" \ + --aggregate-splices "$((splices * 4))" \ + --aggregate-reader-splices "$([[ "${phase}" = launch ]] && echo 5 || echo 0)" \ + --aggregate-reader-bytes "$([[ "${phase}" = launch ]] && echo 12582912 || echo 0)" \ + --required-lease-horizons 2 \ + --splice-ramp-seconds 120 \ + --max-generator-rss-growth-mib 512 \ + --ramp-seconds 180 \ + --duration-seconds 210 \ + --shard-count 4 \ + --shard-index "${shard}" \ + > "${proof_dir}/${phase}-${shard}.jsonl" & + pids+=("$!") + done + failed=0 + for pid in "${pids[@]}"; do + if ! wait "${pid}"; then failed=1; break; fi + done + if test "${failed}" = 1; then + stop_shards + for shard in 0 1 2 3; do + jq -cer 'select(.event == "relay_load_progress" or .event == "relay_load_complete") | + {event, shardIndex, active, peakActive, connected, connectionFailures, + rampConnectionFailures, connectionFailuresByReason, unexpectedCloses, + protocolErrors, refreshErrors, socketErrors, elapsedSeconds}' \ + "${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 || true + done + trap - EXIT + return 1 + fi + trap - EXIT + for shard in 0 1 2 3; do + jq -cer 'select(.event == "relay_load_complete")' \ + "${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 + done | jq -s . > "${proof_dir}/${phase}.json" + } + run_phase launch 5 5 + + - name: Collect and validate aggregate staging proof evidence + env: + PROOF_STARTED_AT: ${{ steps.promote.outputs.started_at }} + shell: bash + run: | + set -euo pipefail + proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof" + ended_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + sleep 60 + gcloud logging read \ + "timestamp>=\"${PROOF_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \ + --project "${GCP_PROJECT_ID}" --limit 20000 --format json \ + > "${proof_dir}/runtime-metrics.json" + node dev/scripts/relay-asia-rollout-evidence.mjs create-staging \ + --repository "${GITHUB_REPOSITORY}" \ + --run-id "${GITHUB_RUN_ID}" \ + --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --commit-sha "${GITHUB_SHA}" \ + --image-digest "${IMAGE_DIGEST}" \ + --selector-generation "${{ steps.promote.outputs.generation }}" \ + --started-at "${PROOF_STARTED_AT}" \ + --ended-at "${ended_at}" \ + --launch-report "${proof_dir}/launch.json" \ + --logs-json "${proof_dir}/runtime-metrics.json" \ + --output "${proof_dir}/evidence.json" + + - name: Return staging C4 to migration-only + if: ${{ always() && steps.promote.outcome != 'skipped' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging \ + --mode recover-promotion \ + --cell-ids staging-gce-c4 \ + --expected-generation "${INITIAL_SELECTOR_GENERATION}" \ + --attempt-id "${PROMOTE_ATTEMPT_ID}" \ + --image-digest "${IMAGE_DIGEST}")" + if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi + promoted_generation="$(jq -er '.generation' <<< "${promoted}")" + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging \ + --mode rollback \ + --cell-ids staging-gce-c4 \ + --expected-generation "${promoted_generation}" \ + --attempt-id "${ROLLBACK_ATTEMPT_ID}" \ + --image-digest "${IMAGE_DIGEST}")" + test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only + + - name: Upload immutable staging readiness evidence + if: ${{ success() }} + uses: actions/upload-artifact@v4 + with: + name: relay-asia-staging-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/relay-asia-staging-proof/evidence.json + if-no-files-found: error + retention-days: 7 + + recover: + if: ${{ always() && github.ref == 'refs/heads/main' }} + needs: prove + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 15 + environment: staging + env: + IMAGE_DIGEST: ${{ inputs.image-digest }} + INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }} + PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }} + ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }} + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Recover staging C4 with a fresh identity + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging \ + --mode recover-promotion \ + --cell-ids staging-gce-c4 \ + --expected-generation "${INITIAL_SELECTOR_GENERATION}" \ + --attempt-id "${PROMOTE_ATTEMPT_ID}" \ + --image-digest "${IMAGE_DIGEST}")" + if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi + promoted_generation="$(jq -er '.generation' <<< "${promoted}")" + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging \ + --mode rollback \ + --cell-ids staging-gce-c4 \ + --expected-generation "${promoted_generation}" \ + --attempt-id "${ROLLBACK_ATTEMPT_ID}" \ + --image-digest "${IMAGE_DIGEST}")" + test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only diff --git a/.github/workflows/cloud-prove-relay-staging-capacity.yml b/.github/workflows/cloud-prove-relay-staging-capacity.yml new file mode 100644 index 00000000000..ee4fb5ed9b1 --- /dev/null +++ b/.github/workflows/cloud-prove-relay-staging-capacity.yml @@ -0,0 +1,917 @@ +name: Prove Relay Staging Capacity + +on: + workflow_dispatch: + inputs: + mode: + description: Verify or change C3 capacity, restore admission, or refresh empty Asia C4 + required: true + default: verify + type: choice + options: + - verify + - apply + - restore-admission + - refresh-asia-c4-image + expected-hard-cap: + description: Exact cap declared for staging-gce-c3 in the reviewed staging tfvars + required: true + default: '600' + type: choice + options: + - '1000' + - '600' + expected-unobserved-bound: + description: Exact bound declared for staging-gce-c3 in the reviewed staging tfvars + required: true + default: '60' + type: choice + options: + - '60' + - '0' + confirmation: + description: Exact confirmation required for a mutation + required: false + type: string + expected-selector-generation: + description: Exact staging selector generation for a C4 image refresh + required: false + type: string + predecessor-image-digest: + description: Exact current C4 sha256 digest + required: false + type: string + target-image-digest: + description: Exact desired C4 sha256 digest + required: false + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: relay-staging-mutation + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + capacity: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && inputs.mode != 'refresh-asia-c4-image' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: staging + env: + GCP_PROJECT_ID: onorca-cloud-staging + GCP_REGION: ${{ vars.STAGING_GCP_REGION }} + DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging + DIRECTOR_ORIGIN: https://relay-staging.onorca.dev + CELL_ORIGIN: https://c3.relay-staging.onorca.dev + TARGET_CELL_ID: staging-gce-c3 + FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev + FALLBACK_CELL_ID: staging-gce-c2 + CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + EXPECTED_HARD_CAP: ${{ inputs.expected-hard-cap }} + EXPECTED_UNOBSERVED_BOUND: ${{ inputs.expected-unobserved-bound }} + steps: + - uses: actions/checkout@v4 + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: hashicorp/setup-terraform@v3 + if: ${{ inputs.mode != 'restore-admission' }} + with: + terraform_wrapper: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Initialize the exact staging backend + if: ${{ inputs.mode != 'restore-admission' }} + run: node dev/scripts/infra.mjs init --env staging + + - name: Require reviewed desired capacity and image + if: ${{ inputs.mode != 'restore-admission' }} + shell: bash + run: | + CAP_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_hard_cap" + BOUND_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_unobserved_bound" + IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image" + ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone" + DESIRED_CAP="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars <<< "${CAP_EXPRESSION}")" + DESIRED_BOUND="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars <<< "${BOUND_EXPRESSION}")" + DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars <<< "${IMAGE_EXPRESSION}" | jq -r '.')" + TARGET_ZONE="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars <<< "${ZONE_EXPRESSION}" | jq -r '.')" + MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ + | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')" + DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars \ + <<< 'local.relay_director_cells_json' | jq -r '.')" + CELL_ORIGIN="$(jq -r --arg cell "${TARGET_CELL_ID}" \ + '.[] | select(.id == $cell) | .url' <<< "${DESIRED_CELLS_JSON}")" + test "${DESIRED_CAP}" = "${EXPECTED_HARD_CAP}" + test "${DESIRED_BOUND}" = "${EXPECTED_UNOBSERVED_BOUND}" + [[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]] + [[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]] + [[ "${MIG_NAME}" =~ ^[a-z0-9-]+$ ]] + test "${CELL_ORIGIN}" = "https://c3.relay-staging.onorca.dev" + jq -e \ + --arg cell "${TARGET_CELL_ID}" \ + --arg fallback "${FALLBACK_CELL_ID}" \ + --argjson cap "${EXPECTED_HARD_CAP}" \ + --argjson bound "${EXPECTED_UNOBSERVED_BOUND}" \ + '(any(.[]; .id == $cell and .connectionHardCap == $cap and .connectionUnobservedBound == $bound)) and + (any(.[]; .id == $fallback and .connectionHardCap == 600 and .connectionUnobservedBound == 60))' \ + <<< "${DESIRED_CELLS_JSON}" >/dev/null + echo "DESIRED_IMAGE=${DESIRED_IMAGE}" >> "${GITHUB_ENV}" + echo "TARGET_ZONE=${TARGET_ZONE}" >> "${GITHUB_ENV}" + echo "MIG_NAME=${MIG_NAME}" >> "${GITHUB_ENV}" + echo "CELL_ORIGIN=${CELL_ORIGIN}" >> "${GITHUB_ENV}" + echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}" + + - name: Verify exact compatible director image + if: ${{ inputs.mode != 'restore-admission' }} + shell: bash + run: | + SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" + ACTIVE_REVISION="$(jq -r \ + '[.status.traffic[] | select((.percent // 0) > 0)] + | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \ + <<< "${SERVICE_JSON}")" + test -n "${ACTIVE_REVISION}" + ACTIVE_IMAGE="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --format='value(spec.containers[0].image)')" + test "${ACTIVE_IMAGE}" = "${DESIRED_IMAGE}" + echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" >> "${GITHUB_ENV}" + + - name: Require explicit transition confirmation + if: ${{ inputs.mode == 'apply' }} + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: test "${CONFIRMATION}" = "FENCE_AND_TRANSITION_STAGING_C3" + + - name: Require explicit admission restore confirmation + if: ${{ inputs.mode == 'restore-admission' }} + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: test "${CONFIRMATION}" = "RESTORE_STAGING_C2_C3_GENERAL" + + - name: Require capacity identity and exact predecessor state + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + case "${EXPECTED_HARD_CAP}/${EXPECTED_UNOBSERVED_BOUND}" in + 1000/0) + PREDECESSOR_C3_CAP=600 + PREDECESSOR_C3_BOUND=60 + ;; + 1000/60) + PREDECESSOR_C3_CAP=1000 + PREDECESSOR_C3_BOUND=0 + ;; + 600/60) + PREDECESSOR_C3_CAP=1000 + PREDECESSOR_C3_BOUND=60 + ;; + *) + echo "Unsupported staging capacity transition" >&2 + exit 1 + ;; + esac + ACTIVE_REVISION="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -r ' + [.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" + test -n "${ACTIVE_REVISION}" + CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -cer ' + [.spec.containers[0].env[]? | + select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | + if length == 1 then .[0] | fromjson else error("missing director topology") end')" + PREDECESSOR_CELLS_JSON="$(jq -ce \ + --arg cell "${TARGET_CELL_ID}" \ + --argjson cap "${PREDECESSOR_C3_CAP}" \ + --argjson bound "${PREDECESSOR_C3_BOUND}" \ + 'map(if .id == $cell then . + { + connectionHardCap: $cap, + connectionUnobservedBound: $bound + } else . end)' <<< "${DESIRED_CELLS_JSON}")" + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${FALLBACK_CELL_ORIGIN}" \ + --cell-id "${FALLBACK_CELL_ID}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission either \ + --draining forbidden \ + --activity allowed + if jq -ne \ + --argjson current "${CURRENT_CELLS_JSON}" \ + --argjson expected "${DESIRED_CELLS_JSON}" \ + '$current == $expected'; then + if node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed; then + TRANSITION_PHASE=cell-active + elif node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission migration-only \ + --draining forbidden \ + --activity allowed; then + TRANSITION_PHASE=cell-ready + else + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --heartbeat either \ + --admission migration-only \ + --draining required \ + --activity restart-safe + TRANSITION_PHASE=director-ready + fi + else + jq -ne \ + --argjson current "${CURRENT_CELLS_JSON}" \ + --argjson expected "${PREDECESSOR_CELLS_JSON}" \ + '$current == $expected' + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${PREDECESSOR_C3_CAP}" \ + --unobserved-bound "${PREDECESSOR_C3_BOUND}" \ + --heartbeat fresh \ + --admission either \ + --draining either \ + --activity allowed + TRANSITION_PHASE=predecessor + fi + echo "TRANSITION_PHASE=${TRANSITION_PHASE}" >> "${GITHUB_ENV}" + + - name: Restore C2 as the safe placement fallback + if: ${{ inputs.mode == 'restore-admission' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${FALLBACK_CELL_ORIGIN}" \ + --cell-id "${FALLBACK_CELL_ID}" \ + --heartbeat fresh \ + --admission either \ + --draining forbidden \ + --activity allowed + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode restore-fallback \ + --general-cell-ids "${FALLBACK_CELL_ID}" + + - name: Require a healthy non-draining C3 before restoring it + if: ${{ inputs.mode == 'restore-admission' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission either \ + --draining forbidden \ + --activity allowed + + - name: Require a healthy general C2 before isolating C3 + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + if test "${TRANSITION_PHASE}" = cell-active; then + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode restore-fallback \ + --general-cell-ids "${FALLBACK_CELL_ID}" + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${FALLBACK_CELL_ORIGIN}" \ + --cell-id "${FALLBACK_CELL_ID}" \ + --hard-cap 600 \ + --unobserved-bound 60 \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed + + - name: Reversibly isolate and drain C3 + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + if test "${TRANSITION_PHASE}" = cell-ready; then + exit 0 + fi + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode isolate + + - name: Verify restart-safe migration-only target + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + if test "${TRANSITION_PHASE}" = cell-ready; then + exit 0 + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --heartbeat either \ + --admission migration-only \ + --draining required \ + --activity restart-safe + + - name: Verify current capacity + if: ${{ inputs.mode == 'verify' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed + + - name: Deploy reviewed director topology and remove pre-protocol revisions + if: ${{ inputs.mode == 'apply' }} + run: | + if test "${TRANSITION_PHASE}" != predecessor; then + echo "DIRECTOR_CONFIG_CHANGED=false" >> "${GITHUB_ENV}" + exit 0 + fi + RELEASE_ID="capacity-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" + DEPLOY_RESULT="$(node dev/scripts/deploy-relay-blue-green.mjs \ + --project "${GCP_PROJECT_ID}" \ + --region "${GCP_REGION}" \ + --service "${DIRECTOR_SERVICE_NAME}" \ + --image "${ACTIVE_IMAGE}" \ + --role director \ + --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ + --capacity-cell-id "${TARGET_CELL_ID}" \ + --director-cells-json "${DESIRED_CELLS_JSON}" \ + --min-instances 0 \ + --prune-revisions true \ + --release-id "${RELEASE_ID}")" + echo "${DEPLOY_RESULT}" + DIRECTOR_CONFIG_CHANGED="$(jq -r '.topologyChanged' <<< "${DEPLOY_RESULT}")" + [[ "${DIRECTOR_CONFIG_CHANGED}" =~ ^(true|false)$ ]] + echo "DIRECTOR_CONFIG_CHANGED=${DIRECTOR_CONFIG_CHANGED}" >> "${GITHUB_ENV}" + + - name: Require fail-closed director transition + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + if test "${TRANSITION_PHASE}" = cell-ready; then + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission migration-only \ + --draining forbidden \ + --activity allowed + exit 0 + fi + HEARTBEAT_EXPECTATION=either + if test "${DIRECTOR_CONFIG_CHANGED}" = true; then + HEARTBEAT_EXPECTATION=stale + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat "${HEARTBEAT_EXPECTATION}" \ + --admission migration-only \ + --draining required \ + --activity restart-safe + + - name: Plan and apply only the exact empty cell + if: ${{ inputs.mode == 'apply' }} + shell: bash + run: | + recreate_fixed_one_instance() { + local instance + instance="$(gcloud compute instance-groups managed list-instances \ + "${MIG_NAME}" \ + --project "${GCP_PROJECT_ID}" \ + --zone "${TARGET_ZONE}" \ + --format=json \ + | jq -er ' + if length == 1 and .[0].instanceStatus == "RUNNING" and + .[0].currentAction == "NONE" + then .[0].instance | split("/") | last + else error("capacity cell does not have one stable running instance") end')" + gcloud compute instance-groups managed recreate-instances \ + "${MIG_NAME}" \ + --instances "${instance}" \ + --project "${GCP_PROJECT_ID}" \ + --zone "${TARGET_ZONE}" \ + --quiet + } + + terraform -chdir=infra/terraform plan \ + -var-file=environments/staging.tfvars \ + '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c3"]' \ + '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]' \ + -out="${RUNNER_TEMP}/relay-capacity-cell.tfplan" + PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \ + "${RUNNER_TEMP}/relay-capacity-cell.tfplan" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode cell \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --image "${DESIRED_IMAGE}")" + echo "${PLAN_RESULT}" + CELL_PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")" + [[ "${CELL_PLAN_CHANGES}" =~ ^(0|2)$ ]] + if test "${TRANSITION_PHASE}" = cell-ready; then + test "${CELL_PLAN_CHANGES}" = 0 + elif test "${TRANSITION_PHASE}" = cell-active; then + test "${CELL_PLAN_CHANGES}" = 0 + recreate_fixed_one_instance + elif test "${CELL_PLAN_CHANGES}" = 0; then + recreate_fixed_one_instance + else + terraform -chdir=infra/terraform apply \ + -auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan" + fi + gcloud compute instance-groups managed wait-until \ + "${MIG_NAME}" \ + --stable \ + --project "${GCP_PROJECT_ID}" \ + --zone "${TARGET_ZONE}" \ + --timeout 900 + + - name: Verify exact live cap and fresh matching heartbeat + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission migration-only \ + --draining forbidden \ + --activity allowed + + - name: Make C3 the only staging placement cell + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode activate + + - name: Verify the sole general canary after transition + if: ${{ inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed + + - name: Restore the reviewed C2 and C3 placement set + if: ${{ inputs.mode == 'restore-admission' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode restore \ + --general-cell-ids staging-gce-c2,staging-gce-c3 + + - name: Verify restored C3 admission and capacity + if: ${{ inputs.mode == 'restore-admission' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --hard-cap "${EXPECTED_HARD_CAP}" \ + --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ + --heartbeat fresh \ + --admission general \ + --draining forbidden \ + --activity allowed + + - name: Preserve C2 as the safe fallback after a failed transition + if: ${{ failure() && inputs.mode == 'apply' }} + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" \ + --mode restore-fallback \ + --general-cell-ids "${FALLBACK_CELL_ID}" + + refresh-asia-c4-image: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && github.ref == 'refs/heads/main' && inputs.mode == 'refresh-asia-c4-image' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 90 + environment: staging + env: + GCP_PROJECT_ID: onorca-cloud-staging + GCP_REGION: ${{ vars.STAGING_GCP_REGION }} + DIRECTOR_ORIGIN: https://relay-staging.onorca.dev + CELL_ORIGIN: https://c4.relay-staging.onorca.dev + TARGET_CELL_ID: staging-gce-c4 + EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} + PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} + TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }} + APPROVED_PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7 + steps: + - uses: actions/checkout@v4 + + - name: Require the exact bounded C4 refresh + env: + CONFIRMATION: ${{ inputs.confirmation }} + shell: bash + run: | + set -euo pipefail + test "${CONFIRMATION}" = REFRESH_STAGING_ASIA_C4_IMAGE + [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] + [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + [[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + test "${PREDECESSOR_IMAGE_DIGEST}" != "${TARGET_IMAGE_DIGEST}" + test "${PREDECESSOR_IMAGE_DIGEST}" = "${APPROVED_PREDECESSOR_IMAGE_DIGEST}" + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.15.8 + terraform_wrapper: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Initialize the exact staging backend + run: node dev/scripts/infra.mjs init --env staging + + - name: Resolve the reviewed C4 image and shape + shell: bash + run: | + set -euo pipefail + cells="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars -var manage_artifact_dns=false \ + <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" + shape="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${cells}")" + test "$(jq -r '.hostname' <<< "${shape}")" = c4 + test "$(jq -r '.region' <<< "${shape}")" = asia-east2 + test "$(jq -r '.zone' <<< "${shape}")" = asia-east2-a + test "$(jq -r '.machine_type' <<< "${shape}")" = e2-standard-4 + test "$(jq -r '.capacity_requests' <<< "${shape}")" = 6000 + test "$(jq -r '.database_pool_max' <<< "${shape}")" = 10 + test "$(jq -r '.connection_hard_cap' <<< "${shape}")" = 3000 + test "$(jq -r '.connection_unobserved_bound' <<< "${shape}")" = 60 + test "$(jq -r '.initially_enabled' <<< "${shape}")" = false + desired_image="$(jq -r '.image' <<< "${shape}")" + test "${desired_image}" = \ + "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${TARGET_IMAGE_DIGEST}" + served_digest="$(gcloud artifacts docker images describe "${desired_image}" \ + --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" + test "${served_digest}" = "${TARGET_IMAGE_DIGEST}" + mig_name="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ + | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')" + test "${mig_name}" = orca-cloud-staging-relay-gce-c4 + { + echo "DESIRED_IMAGE=${desired_image}" + echo "ROLLBACK_IMAGE=us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${PREDECESSOR_IMAGE_DIGEST}" + echo "TARGET_ZONE=asia-east2-a" + echo "MIG_NAME=${mig_name}" + } >> "${GITHUB_ENV}" + + - name: Save, validate, and classify the exact C4 plan + id: plan + shell: bash + run: | + set -euo pipefail + plan="${RUNNER_TEMP}/relay-c4-image-refresh.tfplan" + terraform -chdir=infra/terraform plan \ + -var-file=environments/staging.tfvars -var manage_artifact_dns=false \ + '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ + '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ + -out="${plan}" + result="$(terraform -chdir=infra/terraform show -json "${plan}" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ + --unobserved-bound 60 --image "${DESIRED_IMAGE}" \ + --rollback-image "${ROLLBACK_IMAGE}")" + case "$(jq -r '[.changes,.changeKind] | join(":")' <<< "${result}")" in + 2:replacement) refresh_phase=predecessor ;; + 0:none|*:obsolete-template-delete) refresh_phase=applied ;; + *:manager-convergence|*:replacement-with-obsolete-template) refresh_phase=converging ;; + *) exit 1 ;; + esac + { + echo "PLAN_CHANGES=$(jq -r '.changes' <<< "${result}")" + echo "REFRESH_PHASE=${refresh_phase}" + } >> "${GITHUB_ENV}" + + - id: state-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Verify the exact selector and current C4 state + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.state-auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \ + --expected-generation '' --expected-membership-sha256 '' --attempt-id '' \ + --image-digest "${TARGET_IMAGE_DIGEST}")" + test "$(jq -r '.generation' <<< "${inspect}")" = "${EXPECTED_SELECTOR_GENERATION}" + test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \ + migration-only + expected_digests="${TARGET_IMAGE_DIGEST}" + if test "${REFRESH_PHASE}" = predecessor; then + expected_digests="${PREDECESSOR_IMAGE_DIGEST}" + elif test "${REFRESH_PHASE}" = converging; then + expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}" + fi + if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \ + "${CELL_ORIGIN}/v1/admin/runtime-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data '{"v":1}')"; then + current_digest="$(jq -er '.imageDigest' <<< "${runtime}")" + case ",${expected_digests}," in + *,"${current_digest}",*) ;; + *) exit 1 ;; + esac + source_incarnation='' + draining=forbidden + if test "${REFRESH_PHASE}" != applied; then + status="$(curl --fail-with-body --max-time 30 --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + source_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")" + [[ "${source_incarnation}" =~ ^[0-9a-f-]{36}$ ]] + if test "$(jq -r '.draining' <<< "${runtime}")" = true; then + draining=required + fi + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ + --heartbeat fresh --admission migration-only --draining "${draining}" \ + --activity quiescent --expected-image-digests "${expected_digests}" + runtime_available=true + else + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \ + --admission migration-only --draining either --activity restart-safe \ + --timeout-ms 300000 + source_incarnation='' + runtime_available=false + fi + { + echo "MIG_STABLE_AT_MS=0" + echo "MUTATION_STARTED=false" + echo "REPLACEMENT_STARTED_AT_MS=0" + echo "RUNTIME_AVAILABLE=${runtime_available}" + echo "SOURCE_INCARNATION=${source_incarnation}" + } >> "${GITHUB_ENV}" + + - id: fence-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Fence C4 and prove it stayed empty before replacement + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + if test "${REFRESH_PHASE}" = applied; then exit 0; fi + echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" + if test "${RUNTIME_AVAILABLE}" = false; then exit 0; fi + node dev/scripts/prepare-relay-capacity-canary.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --mode isolate + fence_digests="${PREDECESSOR_IMAGE_DIGEST}" + if test "${REFRESH_PHASE}" = converging; then + fence_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}" + fi + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ + --heartbeat fresh --admission migration-only --draining required \ + --activity quiescent --expected-image-digests "${fence_digests}" + + - name: Apply the exact saved C4 plan + shell: bash + run: | + set -euo pipefail + if test "${PLAN_CHANGES}" = 0 && test "${RUNTIME_AVAILABLE}" = true; then exit 0; fi + if test "${REFRESH_PHASE}" = applied && test "${RUNTIME_AVAILABLE}" = false; then + echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" + fi + if test "${REFRESH_PHASE}" != applied; then + replacement_started_at_ms="$(date -u +%s%3N)" + echo "REPLACEMENT_STARTED_AT_MS=${replacement_started_at_ms}" >> "${GITHUB_ENV}" + fi + if test "${PLAN_CHANGES}" != 0; then + terraform -chdir=infra/terraform apply -auto-approve \ + "${RUNNER_TEMP}/relay-c4-image-refresh.tfplan" + fi + gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 + if test "${RUNTIME_AVAILABLE}" = false && test "${REFRESH_PHASE}" = applied; then + instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \ + | jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and + .[0].currentAction == "NONE" then .[0].instance | split("/") | last + else error("C4 is not one stable running instance") end')" + gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \ + --instances "${instance}" --project "${GCP_PROJECT_ID}" \ + --zone "${TARGET_ZONE}" --quiet + gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 + fi + echo "MIG_STABLE_AT_MS=$(date -u +%s%3N)" >> "${GITHUB_ENV}" + + - id: post-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Verify new C4 incarnation, image, and unchanged isolation + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ + --heartbeat fresh --admission migration-only --draining forbidden \ + --activity quiescent --expected-image-digests "${TARGET_IMAGE_DIGEST}" \ + --timeout-ms 240000 + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \ + --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ + --expected-membership-sha256 '' --attempt-id '' \ + --image-digest "${TARGET_IMAGE_DIGEST}")" + test "$(jq -r '.generation' <<< "${result}")" = "${EXPECTED_SELECTOR_GENERATION}" + test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \ + migration-only + for _ in $(seq 1 36); do + status="$(curl --fail-with-body --max-time 30 --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \ + test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ + -ge "${MIG_STABLE_AT_MS}"; then break; fi + sleep 5 + done + target_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")" + test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true + test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ + -ge "${MIG_STABLE_AT_MS}" + if test "${REFRESH_PHASE}" != applied; then + if test -n "${SOURCE_INCARNATION}"; then + test "${target_incarnation}" != "${SOURCE_INCARNATION}" + fi + test "$(jq -r '.status.runtime.startedAt' <<< "${status}")" \ + -ge "${REPLACEMENT_STARTED_AT_MS}" + fi + + - name: Require an empty targeted Terraform readback + shell: bash + run: | + set -euo pipefail + plan="${RUNNER_TEMP}/relay-c4-image-readback.tfplan" + terraform -chdir=infra/terraform plan \ + -var-file=environments/staging.tfvars -var manage_artifact_dns=false \ + '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ + '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ + -out="${plan}" + result="$(terraform -chdir=infra/terraform show -json "${plan}" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ + --unobserved-bound 60 --image "${DESIRED_IMAGE}" \ + --rollback-image "${ROLLBACK_IMAGE}")" + test "$(jq -r '.changes' <<< "${result}")" = 0 diff --git a/.github/workflows/cloud-publish-relay-production.yml b/.github/workflows/cloud-publish-relay-production.yml new file mode 100644 index 00000000000..d7bb42b4c55 --- /dev/null +++ b/.github/workflows/cloud-publish-relay-production.yml @@ -0,0 +1,131 @@ +name: Publish Relay Production Image + +on: + workflow_dispatch: + inputs: + mode: + description: Publish a new production image or mirror an existing immutable image to staging + required: true + default: publish + type: choice + options: [publish, mirror-staging] + image-digest: + description: Exact existing production digest for mirror-staging mode + required: false + type: string + confirmation: + description: Enter MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING for mirror-staging mode + required: false + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: publish-relay-production + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + publish: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + environment: production + env: + GCP_PROJECT_ID: onorca-cloud + GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} + REPOSITORY_ID: orca-cloud + IMAGE_NAME: relay + PUBLISH_MODE: ${{ inputs.mode }} + MIRROR_DIGEST: ${{ inputs.image-digest }} + MIRROR_CONFIRMATION: ${{ inputs.confirmation }} + steps: + - uses: actions/checkout@v4 + + - name: Validate the exact publish request before authentication + shell: bash + run: | + set -euo pipefail + if test "${PUBLISH_MODE}" = mirror-staging; then + [[ "${MIRROR_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] + test "${MIRROR_CONFIRMATION}" = MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING + else + test "${PUBLISH_MODE}" = publish + test -z "${MIRROR_DIGEST}" + test -z "${MIRROR_CONFIRMATION}" + fi + + - uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: docker/setup-buildx-action@v3 + + - name: Configure Docker auth + run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet + + - name: Build and publish immutable image + if: ${{ inputs.mode == 'publish' }} + run: | + IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${GITHUB_SHA}" + docker build -f apps/relay/Dockerfile -t "${IMAGE_TAG}" . + docker push "${IMAGE_TAG}" + DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')" + test -n "${DIGEST}" + IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${DIGEST}" + { + echo '### Terraform candidate image' + echo + echo "\`${IMAGE}\`" + echo + echo 'Declare this digest on a distinct disabled candidate cell in a reviewed Terraform PR.' + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Build and publish immutable fence broker + if: ${{ inputs.mode == 'publish' }} + run: | + IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker:sha-${GITHUB_SHA}" + docker build \ + --build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=${GITHUB_SHA}" \ + -f apps/relay-fence-broker/Dockerfile \ + -t "${IMAGE_TAG}" . + docker push "${IMAGE_TAG}" + DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')" + test -n "${DIGEST}" + IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker@${DIGEST}" + { + echo + echo '### Terraform fence broker image' + echo + echo "\`${IMAGE}\`" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Mirror the exact production manifest to staging + if: ${{ inputs.mode == 'mirror-staging' }} + shell: bash + run: | + set -euo pipefail + source_image="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${MIRROR_DIGEST}" + target_tag="${GCP_REGION}-docker.pkg.dev/onorca-cloud-staging/${REPOSITORY_ID}/${IMAGE_NAME}:production-${MIRROR_DIGEST#sha256:}" + source_digest="$(gcloud artifacts docker images describe "${source_image}" \ + --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" + test "${source_digest}" = "${MIRROR_DIGEST}" + docker pull "${source_image}" + docker tag "${source_image}" "${target_tag}" + docker push "${target_tag}" + target_digest="$(gcloud artifacts docker images describe "${target_tag}" \ + --project onorca-cloud-staging --format='value(image_summary.digest)')" + test "${target_digest}" = "${MIRROR_DIGEST}" + { + echo '### Mirrored Relay image' + echo + printf 'Production and staging now resolve the same immutable digest: %s.\n' \ + "${MIRROR_DIGEST}" + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-recover-relay-staging-c4-image.yml b/.github/workflows/cloud-recover-relay-staging-c4-image.yml new file mode 100644 index 00000000000..42cbaf0e374 --- /dev/null +++ b/.github/workflows/cloud-recover-relay-staging-c4-image.yml @@ -0,0 +1,411 @@ +name: Recover Relay Staging C4 Image + +on: + workflow_run: + workflows: [Prove Relay Staging Capacity] + types: [completed] + workflow_dispatch: + inputs: + confirmation: + description: Enter RECOVER_STAGING_ASIA_C4_IMAGE + required: true + type: string + +permissions: + actions: read + contents: read + id-token: write + +defaults: + run: + working-directory: cloud + +jobs: + gate: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion != 'success')) }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 5 + outputs: + recover: ${{ steps.trigger.outputs.recover }} + steps: + - name: Bind recovery to the exact failed C4 job + working-directory: . + id: trigger + env: + CONFIRMATION: ${{ inputs.confirmation }} + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + SOURCE_RUN_EVENT: ${{ github.event.workflow_run.event }} + shell: bash + run: | + set -euo pipefail + if test "${GITHUB_EVENT_NAME}" = workflow_dispatch; then + test "${CONFIRMATION}" = RECOVER_STAGING_ASIA_C4_IMAGE + echo "recover=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + test "${SOURCE_RUN_EVENT}" = workflow_dispatch + [[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]] + jobs="$(gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")" + count="$(jq -s '[.[].jobs[] | select(.name == "refresh-asia-c4-image" and + (.conclusion == "failure" or .conclusion == "cancelled" or + .conclusion == "timed_out"))] | length' <<< "${jobs}")" + if test "${count}" = 0; then + echo "recover=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + test "${count}" = 1 + echo "recover=true" >> "${GITHUB_OUTPUT}" + + recover: + needs: gate + if: ${{ needs.gate.outputs.recover == 'true' }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 90 + environment: staging + concurrency: + group: relay-staging-mutation + cancel-in-progress: false + env: + GCP_PROJECT_ID: onorca-cloud-staging + DIRECTOR_ORIGIN: https://relay-staging.onorca.dev + CELL_ORIGIN: https://c4.relay-staging.onorca.dev + TARGET_CELL_ID: staging-gce-c4 + TARGET_ZONE: asia-east2-a + MIG_NAME: orca-cloud-staging-relay-gce-c4 + PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7 + TARGET_IMAGE_DIGEST: sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563 + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - uses: ./.github/actions/cloud-sql-rollout-lease + with: + bucket: onorca-cloud-staging-terraform-state + object: terraform/state/cloud-sql-rollout/staging.lock + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.15.8 + terraform_wrapper: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Initialize the exact staging backend + run: node dev/scripts/infra.mjs init --env staging + + - id: preflight-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Inspect the exact C4 recovery state + id: preflight + timeout-minutes: 3 + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.preflight-auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \ + --expected-generation '' --expected-membership-sha256 '' --attempt-id '' \ + --image-digest "${PREDECESSOR_IMAGE_DIGEST}")" + generation="$(jq -er '.generation' <<< "${inspect}")" + [[ "${generation}" =~ ^(0|[1-9][0-9]*)$ ]] + test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \ + migration-only + echo "selector_generation=${generation}" >> "${GITHUB_OUTPUT}" + if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \ + "${CELL_ORIGIN}/v1/admin/runtime-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data '{"v":1}')"; then + current_digest="$(jq -er '.imageDigest' <<< "${runtime}")" + [[ "${current_digest}" =~ ^sha256:[a-f0-9]{64}$ ]] + status="$(curl --fail-with-body --max-time 30 --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + jq -e '.draining | type == "boolean"' <<< "${runtime}" >/dev/null + { + echo "runtime_available=true" + echo "current_digest=${current_digest}" + echo "draining=$(jq -r '.draining' <<< "${runtime}")" + echo "ready=$(jq -r '.status.runtime.ready == true' <<< "${status}")" + } >> "${GITHUB_OUTPUT}" + else + echo "runtime_available=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Classify both exact recovery end states + id: recovery-plan + timeout-minutes: 10 + shell: bash + run: | + set -euo pipefail + image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay" + predecessor_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}" + target_image="${image_repository}@${TARGET_IMAGE_DIGEST}" + served_digest="$(gcloud artifacts docker images describe "${predecessor_image}" \ + --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" + test "${served_digest}" = "${PREDECESSOR_IMAGE_DIGEST}" + cells="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars -var manage_artifact_dns=false \ + <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" + test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].image' <<< "${cells}")" = \ + "${target_image}" + target_plan="${RUNNER_TEMP}/relay-c4-image-target.tfplan" + terraform -chdir=infra/terraform plan \ + -var-file=environments/staging.tfvars -var manage_artifact_dns=false -lock-timeout=5m \ + '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ + '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ + -out="${target_plan}" + target_result="$(terraform -chdir=infra/terraform show -json "${target_plan}" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ + --unobserved-bound 60 --image "${target_image}" \ + --rollback-image "${predecessor_image}")" + target_state="$(jq -r '[.changes,.changeKind] | join(":")' <<< "${target_result}")" + case "${target_state}" in + 0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;; + *) exit 1 ;; + esac + recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" \ + --arg image "${predecessor_image}" '.[$cell].image = $image' <<< "${cells}")" + jq -n --argjson cells "${recovery_cells}" \ + '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" + plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan" + terraform -chdir=infra/terraform plan \ + -var-file=environments/staging.tfvars \ + -var-file="${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" \ + -var manage_artifact_dns=false -lock-timeout=5m \ + '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ + '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ + -out="${plan}" + result="$(terraform -chdir=infra/terraform show -json "${plan}" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ + --unobserved-bound 60 --image "${predecessor_image}" \ + --rollback-image "${target_image}")" + changes="$(jq -r '.changes' <<< "${result}")" + change_kind="$(jq -r '.changeKind' <<< "${result}")" + case "${changes}:${change_kind}" in + 0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;; + *) exit 1 ;; + esac + mig="$(gcloud compute instance-groups managed describe "${MIG_NAME}" \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)" + mig_stable="$(jq -r '.status.isStable == true and .status.versionTarget.isReached == true' \ + <<< "${mig}")" + instances="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)" + instance_stable="$(jq -r 'length == 1 and .[0].instanceStatus == "RUNNING" and + .[0].currentAction == "NONE"' <<< "${instances}")" + action=rollback-predecessor + recovery_digest="${PREDECESSOR_IMAGE_DIGEST}" + if test "${{ steps.preflight.outputs.runtime_available }}" = true && \ + test "${{ steps.preflight.outputs.ready }}" = true && \ + test "${{ steps.preflight.outputs.draining }}" = false && \ + test "${mig_stable}" = true && test "${instance_stable}" = true; then + if test "${{ steps.preflight.outputs.current_digest }}" = "${TARGET_IMAGE_DIGEST}" && \ + test "${target_state}" = 0:none; then + action=verify-target + recovery_digest="${TARGET_IMAGE_DIGEST}" + elif test "${{ steps.preflight.outputs.current_digest }}" = \ + "${PREDECESSOR_IMAGE_DIGEST}" && test "${changes}:${change_kind}" = 0:none; then + action=verify-predecessor + fi + fi + { + echo "changes=${changes}" + echo "change_kind=${change_kind}" + echo "action=${action}" + echo "recovery_digest=${recovery_digest}" + } >> "${GITHUB_OUTPUT}" + + - id: fence-auth + if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Fence C4 before predecessor recovery + if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} + timeout-minutes: 6 + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }} + shell: bash + run: | + set -euo pipefail + expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}" + current_digest="${{ steps.preflight.outputs.current_digest }}" + if test -n "${current_digest}" && [[ ",${expected_digests}," != *",${current_digest},"* ]]; then + expected_digests="${expected_digests},${current_digest}" + fi + if curl --silent --show-error --fail-with-body --max-time 30 --request POST \ + "${CELL_ORIGIN}/v1/admin/drain" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data '{"v":1,"graceMs":0}' \ + >/dev/null; then + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ + --heartbeat fresh --admission migration-only --draining required \ + --activity quiescent \ + --expected-image-digests "${expected_digests}" \ + --timeout-ms 240000 + else + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \ + --admission migration-only --draining either --activity restart-safe \ + --timeout-ms 240000 + fi + + - name: Apply and stabilize the saved predecessor plan + id: apply + if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} + timeout-minutes: 20 + env: + CHANGES: ${{ steps.recovery-plan.outputs.changes }} + CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }} + shell: bash + run: | + set -euo pipefail + plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan" + if test "${CHANGES}" != 0; then + terraform -chdir=infra/terraform apply -auto-approve "${plan}" + fi + gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 + echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}" + + - name: Restart only when the plan did not replace C4 + id: restore + if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} + timeout-minutes: 20 + env: + APPLY_STABLE_AT_MS: ${{ steps.apply.outputs.stable_at_ms }} + CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }} + shell: bash + run: | + set -euo pipefail + if [[ "${CHANGE_KIND}" =~ ^(replacement|replacement-with-obsolete-template|manager-convergence)$ ]]; then + echo "stable_at_ms=${APPLY_STABLE_AT_MS}" >> "${GITHUB_OUTPUT}" + exit 0 + fi + instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \ + | jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and + .[0].currentAction == "NONE" then .[0].instance | split("/") | last + else error("C4 is not one stable running instance") end')" + gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \ + --instances "${instance}" --project "${GCP_PROJECT_ID}" \ + --zone "${TARGET_ZONE}" --quiet + gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 + echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}" + + - name: Require an empty selected-image recovery readback + timeout-minutes: 8 + env: + RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }} + shell: bash + run: | + set -euo pipefail + image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay" + recovery_image="${image_repository}@${RECOVERY_DIGEST}" + other_image="${image_repository}@${TARGET_IMAGE_DIGEST}" + if test "${RECOVERY_DIGEST}" = "${TARGET_IMAGE_DIGEST}"; then + other_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}" + fi + cells="$(terraform -chdir=infra/terraform console \ + -var-file=environments/staging.tfvars -var manage_artifact_dns=false \ + <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" + recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" --arg image "${recovery_image}" \ + '.[$cell].image = $image' <<< "${cells}")" + jq -n --argjson cells "${recovery_cells}" \ + '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-readback.tfvars.json" + readback="${RUNNER_TEMP}/relay-c4-image-recovery-readback.tfplan" + terraform -chdir=infra/terraform plan \ + -var-file=environments/staging.tfvars \ + -var-file="${RUNNER_TEMP}/relay-c4-readback.tfvars.json" \ + -var manage_artifact_dns=false -lock-timeout=5m \ + '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ + '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ + -out="${readback}" + readback_result="$(terraform -chdir=infra/terraform show -json "${readback}" \ + | node dev/scripts/validate-relay-capacity-plan.mjs \ + --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ + --unobserved-bound 60 --image "${recovery_image}" \ + --rollback-image "${other_image}")" + test "$(jq -r '.changes' <<< "${readback_result}")" = 0 + + - id: verify-auth + if: ${{ always() }} + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain + id_token_include_email: true + + - name: Verify the recovered image and unchanged isolation + if: ${{ always() && steps.verify-auth.outcome == 'success' }} + timeout-minutes: 8 + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.verify-auth.outputs.id_token }} + SELECTOR_GENERATION: ${{ steps.preflight.outputs.selector_generation }} + STABLE_AT_MS: ${{ steps.restore.outputs.stable_at_ms }} + RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }} + shell: bash + run: | + set -euo pipefail + node dev/scripts/verify-relay-capacity-transition.mjs \ + --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ + --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ + --heartbeat fresh --admission migration-only --draining forbidden \ + --activity quiescent --expected-image-digests "${RECOVERY_DIGEST}" \ + --timeout-ms 240000 + stable_at_ms="${STABLE_AT_MS:-0}" + for _ in $(seq 1 36); do + status="$(curl --fail-with-body --max-time 30 --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \ + test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ + -ge "${stable_at_ms}"; then break; fi + sleep 5 + done + test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true + test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ + -ge "${stable_at_ms}" + result="$(node dev/scripts/operate-relay-asia-admission.mjs \ + --environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \ + --expected-generation "${SELECTOR_GENERATION}" \ + --expected-membership-sha256 '' --attempt-id '' \ + --image-digest "${RECOVERY_DIGEST}")" + test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \ + migration-only diff --git a/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml b/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml new file mode 100644 index 00000000000..d92086a620a --- /dev/null +++ b/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml @@ -0,0 +1,60 @@ +name: Requeue Relay Staging C4 Recovery + +on: + workflow_run: + workflows: [Recover Relay Staging C4 Image] + types: [completed] + +permissions: + actions: write + contents: read + +concurrency: + group: relay-staging-c4-recovery-requeue + cancel-in-progress: false + +defaults: + run: + working-directory: cloud + +jobs: + requeue: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion == 'cancelled') }} + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 5 + steps: + - name: Requeue only a cancelled protected recovery job + working-directory: . + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + shell: bash + run: | + set -euo pipefail + [[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]] + jobs="$(gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")" + count="$(jq -s '[.[].jobs[] | select(.name == "recover" and + .conclusion == "cancelled" and .started_at == null)] | length' <<< "${jobs}")" + if test "${count}" = 0; then exit 0; fi + test "${count}" = 1 + runs="$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/cloud-recover-relay-staging-c4-image.yml/runs?branch=main&per_page=100")" + active=0 + while IFS= read -r run_id; do + active_jobs="$(gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?filter=latest")" + if jq -se ' + ([.[].jobs[] | select(.name == "gate" and .conclusion == "success")] | length) == 1 and + ([.[].jobs[] | select(.name == "recover" and .status != "completed")] | length) == 1 + ' <<< "${active_jobs}" >/dev/null; then + active=1 + break + fi + done < <(jq -r --arg source "${SOURCE_RUN_ID}" \ + '.workflow_runs[] | select((.id | tostring) != $source and + .status != "completed") | .id' <<< "${runs}") + if test "${active}" != 0; then exit 0; fi + gh workflow run cloud-recover-relay-staging-c4-image.yml \ + --repo "${GITHUB_REPOSITORY}" --ref main \ + -f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml new file mode 100644 index 00000000000..f0cc2df2bad --- /dev/null +++ b/.github/workflows/cloud-verify.yml @@ -0,0 +1,120 @@ +name: Cloud Verify + +on: + pull_request: + paths: + - cloud/** + - .github/workflows/cloud-*.yml + - .github/actions/cloud-sql-rollout-lease/** + push: + branches: [main] + paths: + - cloud/** + - .github/workflows/cloud-*.yml + - .github/actions/cloud-sql-rollout-lease/** + +permissions: + contents: read + +concurrency: + group: cloud-verify-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + working-directory: cloud + +jobs: + security: + name: Secret scan + runs-on: blacksmith-2vcpu-ubuntu-2204 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Scan Cloud history with Gitleaks + run: >- + docker run --rm + --volume "${GITHUB_WORKSPACE}:/repo:ro" + zricethezav/gitleaks@sha256:cdbb7c955abce02001a9f6c9f602fb195b7fadc1e812065883f695d1eeaba854 + git /repo --config /repo/cloud/.gitleaks.toml + --log-opts="--all -- cloud :(glob).github/workflows/cloud-*.yml .github/actions/cloud-sql-rollout-lease" + + - name: Scan the single-commit Cloud snapshot with TruffleHog + run: >- + docker run --rm + --volume "${GITHUB_WORKSPACE}:/repo:ro" + trufflesecurity/trufflehog@sha256:5dc064868ba7933601b5cbaea6954954d524ddd5dc6222a9667acea70068bf7d + filesystem /repo --no-verification --fail + --include-paths=/repo/cloud/.trufflehog-include-paths.txt + --exclude-paths=/repo/cloud/.trufflehog-exclude-paths.txt + + # Compiles the workspace. No Postgres service: nothing here reaches a + # database, and the service container costs ~13s of startup. + build: + runs-on: blacksmith-4vcpu-ubuntu-2204 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - run: pnpm install --frozen-lockfile + - run: pnpm build + - run: pnpm typecheck + + # Runs in parallel with build. `pnpm test` compiles the one workspace + # package it needs through the relay pretest hook, so it does not depend on + # `pnpm build` having run. + test: + runs-on: blacksmith-4vcpu-ubuntu-2204 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_DB: orca_relay_test + POSTGRES_PASSWORD: relay_test + POSTGRES_USER: relay_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U relay_test -d orca_relay_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + package_json_file: cloud/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - run: pnpm install --frozen-lockfile + - run: pnpm test + + # Fork pull requests reach this job, so it never configures a backend, never plans, and never + # holds a credential. Only the relay root ships here; foundation and apps stay private. + terraform: + runs-on: blacksmith-2vcpu-ubuntu-2204 + steps: + - uses: actions/checkout@v4 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.15.8 + + - run: terraform -chdir=infra/terraform fmt -check -recursive + - run: terraform -chdir=infra/terraform init -backend=false -input=false + - run: terraform -chdir=infra/terraform validate diff --git a/.oxfmtrc.json b/.oxfmtrc.json index b2aa0deabfa..0f27189d7cb 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -3,5 +3,6 @@ "singleQuote": true, "semi": false, "printWidth": 100, - "trailingComma": "none" + "trailingComma": "none", + "ignorePatterns": ["cloud/**", ".github/actions/cloud-sql-rollout-lease/**"] } diff --git a/.oxlintrc.json b/.oxlintrc.json index cae0d09dd58..77a7e43e807 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -174,5 +174,12 @@ } } ], - "ignorePatterns": ["**/node_modules", "**/dist", "**/out", "tests/e2e/.cross-version-checkouts"] + "ignorePatterns": [ + "**/node_modules", + "**/dist", + "**/out", + "cloud/**", + ".github/actions/cloud-sql-rollout-lease/**", + "tests/e2e/.cross-version-checkouts" + ] } diff --git a/README.md b/README.md index 0f99bfc877f..7a3cbe2360c 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,9 @@ Pair with your desktop app to monitor and steer your agents from your phone. Want to contribute or run locally? See our [CONTRIBUTING.md](.github/CONTRIBUTING.md) guide. +The relay that pairs the mobile app with a desktop host is also in this repository under +[`cloud/`](cloud/README.md), with a separate pnpm workspace and setup guide. + Orca contributors diff --git a/cloud/.dockerignore b/cloud/.dockerignore new file mode 100644 index 00000000000..bc93f9a9f1b --- /dev/null +++ b/cloud/.dockerignore @@ -0,0 +1,15 @@ +.git/ +.github/ +.local/ +.terraform/ +node_modules/ +dist/ +coverage/ +.env +.env.local +.env.local.generated +*.log +*.tfplan +*.tfstate +*.tfstate.* + diff --git a/cloud/.editorconfig b/cloud/.editorconfig new file mode 100644 index 00000000000..c814d3002a4 --- /dev/null +++ b/cloud/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true + diff --git a/cloud/.gitignore b/cloud/.gitignore new file mode 100644 index 00000000000..8105ed487f9 --- /dev/null +++ b/cloud/.gitignore @@ -0,0 +1,24 @@ +node_modules/ +dist/ +coverage/ +.turbo/ +.local/ +.env +.env.local +.env.local.generated +*.log + +# Terraform/OpenTofu local state and plans must stay out of git. +**/.terraform/ +*.tfstate +*.tfstate.* +*.tfplan +crash.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Local dev signing key + SQLite live under data/ — never commit (contains a +# private key and dev PII). Prod supplies the key via ORCA_CLOUD_SIGNING_KEY_PEM. +data/ diff --git a/cloud/.gitleaks.toml b/cloud/.gitleaks.toml new file mode 100644 index 00000000000..fc5c725c37d --- /dev/null +++ b/cloud/.gitleaks.toml @@ -0,0 +1,15 @@ +[extend] +useDefault = true + +[[allowlists]] +description = "Explicit Relay test signing key" +regexTarget = "secret" +regexes = ['''^test-assignment-key-with-at-least-32-bytes$'''] + +# The Cloud SQL rollout lease records `owner/repo/run_id` as the holder of a lease. The action's +# unit tests build fixture holders from that shape, which the generic key rule reads as a secret. +[[allowlists]] +description = "Cloud SQL rollout lease holder keys in the action's unit tests" +regexTarget = "secret" +paths = ['''\.github/actions/cloud-sql-rollout-lease/[a-z-]+\.test\.mjs$'''] +regexes = ['''^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/[0-9]+$'''] diff --git a/cloud/.node-version b/cloud/.node-version new file mode 100644 index 00000000000..14db15d775c --- /dev/null +++ b/cloud/.node-version @@ -0,0 +1,2 @@ +24 + diff --git a/cloud/.npmrc b/cloud/.npmrc new file mode 100644 index 00000000000..e05d6820ed5 --- /dev/null +++ b/cloud/.npmrc @@ -0,0 +1,3 @@ +engine-strict=false +package-manager-strict=true + diff --git a/cloud/.trufflehog-exclude-paths.txt b/cloud/.trufflehog-exclude-paths.txt new file mode 100644 index 00000000000..ea20399af05 --- /dev/null +++ b/cloud/.trufflehog-exclude-paths.txt @@ -0,0 +1 @@ +(^|/)cloud/apps/relay/src/postgres-idle-client-error\.test\.ts$ diff --git a/cloud/.trufflehog-include-paths.txt b/cloud/.trufflehog-include-paths.txt new file mode 100644 index 00000000000..abd82ac6354 --- /dev/null +++ b/cloud/.trufflehog-include-paths.txt @@ -0,0 +1,2 @@ +(^|/)cloud/ +(^|/)\.github/workflows/cloud-[^/]+\.yml$ diff --git a/cloud/README.md b/cloud/README.md new file mode 100644 index 00000000000..8ffcd9fa6b3 --- /dev/null +++ b/cloud/README.md @@ -0,0 +1,92 @@ +# Orca Relay + +The relay that connects the Orca mobile app to a desktop host. Phones and +desktops never talk to each other directly: each opens an outbound WebSocket +to a relay cell, the relay pairs the two sessions, and it splices frames +between them. A director assigns hosts to cells and coordinates migrations; +cells carry the user connections. + +This directory is an independent pnpm workspace inside the Orca monorepo. Run +its commands from `cloud/`, not the repository root. The source is covered by +the repository's root [MIT license](../LICENSE). + +## Packages + +- `packages/relay-contract`: the wire contract shared by the relay, the + desktop app, and the mobile app (frame shapes, close codes, admission budgets, + splice state machine). +- `apps/relay`: the relay server. The same image runs as a director or a cell + depending on `ORCA_RELAY_ROLE`. +- `apps/relay-fence-broker`: a private, IAM-only service that owns the durable + mutation lease, the Terraform checkout, and the narrow Compute mutation used + when a registered target is superseded. The workflow that calls it holds read + and invoke rights only, never those mutation permissions. +- `apps/relay-ops`: the relay operations console and the incident monitor + behind `pnpm ops:relay`, `pnpm incident:relay`, and + `pnpm incident:relay-preflight`. + +## Infrastructure and operations + +- `infra/terraform`: the relay Terraform root. It owns the cells, the director, + the shared Cloud SQL instance, DNS, observability, and every GitHub Workload + Identity provider the relay workflows authenticate through. `backend/` holds + the per-environment backend configuration and `environments/` the tfvars. + Drive it through `pnpm infra:init`, `pnpm infra:plan`, and `pnpm infra:apply`. +- `dev/scripts`: the deploy, capacity, admission, rehome, monitoring, and load + scripts the workflows call, plus the contract tests that pin each workflow + and Terraform surface. Run them with `pnpm test`. +- `dev/contracts` and `dev/fixtures`: the checked-in data those contract tests + read, including the Terraform root partition. +- `docs/`: the relay runbooks, capacity-testing guide, incident-monitor + reference, and the workflow variable reference in `docs/relay-workflows.md`. + +## Workflows + +The 24 `.github/workflows/cloud-*.yml` workflows are the relay's deploy and +operate surface: publish and deploy the director, roll GCE cell capacity, +operate Asia admission and regional rehoming, prove staging capacity, monitor +production, and power staging up and down. `.github/actions/cloud-sql-rollout-lease` +is the compare-and-swap lease that serializes every rollout against the shared +Cloud SQL instance. + +Every one of them is inert. Each top-level job is gated on +`vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'`, a repository variable that is +unset here, so the two scheduled triggers and every manual dispatch skip +without running a step. Only the repository owner, holding the GCP identities +these workflows authenticate as, can turn them on. + +`Cloud Verify` is not gated. It builds, typechecks, lints, tests, secret-scans, +and validates the relay Terraform on every change under `cloud/`, and it runs +on fork pull requests, so it configures no backend and holds no credential. + +## What is not here + +The `terraform-foundation` and `terraform-apps` roots and the API and auth +services live in the private `stablyai/orca-cloud` repository. Scripts and +tests that spanned both trees were narrowed to the relay side rather than +carrying a dangling reference. + +## Local development + +```sh +cd cloud +pnpm install +pnpm build +pnpm test +``` + +`pnpm test` runs the SQLite-backed suites. Tests that need PostgreSQL run only +when `ORCA_RELAY_TEST_POSTGRES_URL` points at a disposable PostgreSQL 16 or 17 +database, for example: + +```sh +docker run --rm -d --name orca-relay-pg -e POSTGRES_HOST_AUTH_METHOD=trust \ + -e POSTGRES_DB=orca_relay_test -p 55440:5432 postgres:16-alpine +ORCA_RELAY_TEST_POSTGRES_URL=postgres://postgres@127.0.0.1:55440/orca_relay_test \ + pnpm --filter @orca-cloud/relay test +docker rm -f orca-relay-pg +``` + +Configuration is read from environment variables validated in +`apps/relay/src/config.ts`. `ORCA_RELAY_ASSIGNMENT_SIGNING_KEY` (at least 32 +bytes) is the only required value; everything else has a local default. diff --git a/cloud/apps/relay-fence-broker/Dockerfile b/cloud/apps/relay-fence-broker/Dockerfile new file mode 100644 index 00000000000..6c452dac393 --- /dev/null +++ b/cloud/apps/relay-fence-broker/Dockerfile @@ -0,0 +1,33 @@ +FROM node:24-bookworm-slim AS build +WORKDIR /workspace +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./ +COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json +RUN pnpm install --frozen-lockfile +COPY apps/relay-fence-broker apps/relay-fence-broker +RUN pnpm --filter @orca-cloud/relay-fence-broker build + +FROM hashicorp/terraform:1.15.8 AS terraform + +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:slim +ARG ORCA_RELAY_FENCE_IMAGE_COMMIT +RUN test "$(printf '%s' "${ORCA_RELAY_FENCE_IMAGE_COMMIT}" | grep -E '^[a-f0-9]{40}$')" +ENV NODE_ENV=production +ENV PORT=8080 +ENV ORCA_RELAY_FENCE_IMAGE_COMMIT=${ORCA_RELAY_FENCE_IMAGE_COMMIT} +ENV IAC_TOOL=terraform +WORKDIR /workspace +COPY --from=build /usr/local /usr/local +COPY --from=terraform /bin/terraform /usr/local/bin/terraform +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json +COPY --from=build /workspace/apps/relay-fence-broker/dist apps/relay-fence-broker/dist +COPY dev/scripts dev/scripts +COPY infra/terraform infra/terraform +RUN corepack enable \ + && pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay-fence-broker... \ + && useradd --create-home --uid 10001 broker \ + && chown -R broker:broker /workspace +USER broker +EXPOSE 8080 +CMD ["node", "apps/relay-fence-broker/dist/index.js"] diff --git a/cloud/apps/relay-fence-broker/package.json b/cloud/apps/relay-fence-broker/package.json new file mode 100644 index 00000000000..c9bf65c2cf3 --- /dev/null +++ b/cloud/apps/relay-fence-broker/package.json @@ -0,0 +1,27 @@ +{ + "name": "@orca-cloud/relay-fence-broker", + "private": true, + "version": "0.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "pnpm clean && tsc -p tsconfig.build.json", + "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", + "dev": "tsx watch src/index.ts", + "lint": "tsc -p tsconfig.json --noEmit", + "start": "node dist/index.js", + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "@hono/node-server": "^1.19.14", + "hono": "^4.12.27", + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "tsx": "^4.21.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/apps/relay-fence-broker/src/app.test.ts b/cloud/apps/relay-fence-broker/src/app.test.ts new file mode 100644 index 00000000000..0047c95b07b --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/app.test.ts @@ -0,0 +1,316 @@ +import { describe, expect, it, vi } from 'vitest' +import { createApp } from './app.js' +import type { RelayFenceBrokerConfig } from './config.js' +import type { + GoogleStorageMutationLease, + MutationLease +} from './mutation-lease.js' + +const commit = 'a'.repeat(40) +const config: RelayFenceBrokerConfig = { + port: 8080, + project: 'onorca-cloud', + stateBucket: 'onorca-cloud-terraform-state', + leaseObject: 'terraform/state/relay-fence-broker/production.lock', + directorOrigin: 'https://relay.onorca.dev', + adminAudience: 'https://relay.onorca.dev/v1/admin/drain', + requesterServiceAccount: 'requester@example.com', + runtimeServiceAccount: 'runtime@example.com', + sourceCellId: 'production-gce-c3', + failedTargetCellId: 'production-gce-c11', + replacementTargetCellId: 'production-gce-c12', + imageCommit: commit, + terraformDir: 'infra/terraform', + unobservedConnectionBound: 10, + connectionCeiling: 600 +} + +function request(fenceCommit = commit): Request { + return new Request('http://broker/v1/supersede-target', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + v: 1, + operationId: 'c11-to-c12-forward', + fenceCommit, + confirmation: 'SUPERSEDE_TARGET' + }) + }) +} + +function recoveryRequest(): Request { + return new Request('http://broker/v1/supersede-target', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + v: 1, + operationId: 'c11-to-c12-forward', + fenceCommit: commit, + completedFenceRecovery: { + attemptId: '11111111-1111-4111-8111-111111111111', + fenceCommit: 'b'.repeat(40), + gceOperation: 'operation-1', + terraformStateSerial: 61, + planObjectGeneration: '123', + terraformStateObjectGeneration: '456', + terraformStateObjectSha256: 'c'.repeat(64) + }, + expectedLease: { + generation: '7', + operationId: 'c11-to-c12-forward', + requestDigest: 'd'.repeat(64) + }, + confirmation: 'SUPERSEDE_TARGET' + }) + }) +} + +function leaseTakeoverRequest(): Request { + return new Request('http://broker/v1/supersede-target', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + v: 1, + operationId: 'c11-to-c12-forward', + fenceCommit: commit, + expectedLease: { + generation: '7', + operationId: 'c11-to-c12-forward', + requestDigest: 'd'.repeat(64) + }, + confirmation: 'SUPERSEDE_TARGET' + }) + }) +} + +function sourceFenceRequest( + overrides: Record = {} +): Request { + return new Request('http://broker/v1/fence-source', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + v: 1, + operationId: 'c3-final-fence', + fenceCommit: commit, + targetCellIds: ['production-gce-c7', 'production-gce-c12'], + confirmation: 'FENCE_SOURCE', + ...overrides + }) + }) +} + +describe('relay fence broker', () => { + it('runs only after acquiring the durable lease', async () => { + const events: string[] = [] + const lease = { + acquire: vi.fn(async () => { + events.push('acquire') + return { + generation: '7', + record: {} + } as MutationLease + }), + release: vi.fn(async () => { + events.push('release') + }) + } as unknown as GoogleStorageMutationLease + const app = createApp(config, { + lease, + supersede: async () => { + events.push('supersede') + } + }) + + const response = await app.request(request()) + + expect(response.status).toBe(200) + expect(events).toEqual(['acquire', 'supersede', 'release']) + }) + + it('rejects a commit not bound to the immutable image', async () => { + const lease = { + acquire: vi.fn() + } as unknown as GoogleStorageMutationLease + const app = createApp(config, { lease }) + + const response = await app.request(request('b'.repeat(40))) + + expect(response.status).toBe(409) + expect(lease.acquire).not.toHaveBeenCalled() + }) + + it('retains the lease when supersession fails', async () => { + const lease = { + acquire: vi.fn(async () => ({ generation: '7', record: {} })), + release: vi.fn() + } as unknown as GoogleStorageMutationLease + const app = createApp(config, { + lease, + supersede: async () => { + throw new Error('stopped safely') + } + }) + + const response = await app.request(request()) + + expect(response.status).toBe(500) + expect(lease.release).not.toHaveBeenCalled() + }) + + it('passes exact completed-attempt and live-lease recovery pins', async () => { + const lease = { + acquire: vi.fn(async () => ({ generation: '8', record: {} })), + release: vi.fn() + } as unknown as GoogleStorageMutationLease + const supersede = vi.fn(async () => {}) + const app = createApp(config, { lease, supersede }) + + const response = await app.request(recoveryRequest()) + + expect(response.status).toBe(200) + expect(lease.acquire).toHaveBeenCalledWith( + 'c11-to-c12-forward', + expect.objectContaining({ + completedFenceRecovery: expect.objectContaining({ + terraformStateSerial: 61 + }) + }), + { + generation: '7', + operationId: 'c11-to-c12-forward', + requestDigest: 'd'.repeat(64) + } + ) + expect(supersede).toHaveBeenCalledWith( + config, + expect.objectContaining({ + attemptId: '11111111-1111-4111-8111-111111111111' + }) + ) + }) + + it('conditionally resumes the exact live supersession lease', async () => { + const lease = { + acquire: vi.fn(async () => ({ generation: '8', record: {} })), + release: vi.fn() + } as unknown as GoogleStorageMutationLease + const supersede = vi.fn(async () => {}) + const app = createApp(config, { lease, supersede }) + + const response = await app.request(leaseTakeoverRequest()) + + expect(response.status).toBe(200) + expect(lease.acquire).toHaveBeenCalledWith( + 'c11-to-c12-forward', + expect.objectContaining({ + expectedLease: { + generation: '7', + operationId: 'c11-to-c12-forward', + requestDigest: 'd'.repeat(64) + } + }), + { + generation: '7', + operationId: 'c11-to-c12-forward', + requestDigest: 'd'.repeat(64) + } + ) + expect(supersede).toHaveBeenCalledWith(config, undefined) + }) + + it('rejects a live supersession lease for another operation', async () => { + const lease = { + acquire: vi.fn() + } as unknown as GoogleStorageMutationLease + const app = createApp(config, { lease }) + const request = leaseTakeoverRequest() + const body = (await request.json()) as { + expectedLease: { operationId: string } + } + body.expectedLease.operationId = 'another-operation' + + const response = await app.request( + new Request(request.url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + ) + + expect(response.status).toBe(400) + expect(lease.acquire).not.toHaveBeenCalled() + }) + + it('fences the configured source only after acquiring the durable lease', async () => { + const events: string[] = [] + const expectedLease = { + generation: '6', + operationId: 'c3-final-fence', + requestDigest: 'd'.repeat(64) + } + const lease = { + acquire: vi.fn(async () => { + events.push('acquire') + return { generation: '7', record: {} } as MutationLease + }), + release: vi.fn(async () => { + events.push('release') + }) + } as unknown as GoogleStorageMutationLease + const fenceSource = vi.fn(async () => { + events.push('fence') + }) + const app = createApp(config, { lease, fenceSource }) + + const response = await app.request(sourceFenceRequest({ expectedLease })) + + expect(response.status).toBe(200) + expect(events).toEqual(['acquire', 'fence', 'release']) + expect(lease.acquire).toHaveBeenCalledWith( + 'c3-final-fence', + expect.objectContaining({ targetCellIds: expect.any(Array) }), + expectedLease + ) + expect(fenceSource).toHaveBeenCalledWith(config, [ + 'production-gce-c7', + 'production-gce-c12' + ]) + }) + + it('rejects duplicate targets and the configured source', async () => { + const lease = { acquire: vi.fn() } as unknown as GoogleStorageMutationLease + const app = createApp(config, { lease }) + + const duplicate = await app.request( + sourceFenceRequest({ + targetCellIds: ['production-gce-c7', 'production-gce-c7'] + }) + ) + const source = await app.request( + sourceFenceRequest({ targetCellIds: [config.sourceCellId] }) + ) + + expect(duplicate.status).toBe(400) + expect(source.status).toBe(400) + expect(lease.acquire).not.toHaveBeenCalled() + }) + + it('retains the source-fence lease after a controlled failure', async () => { + const lease = { + acquire: vi.fn(async () => ({ generation: '7', record: {} })), + release: vi.fn() + } as unknown as GoogleStorageMutationLease + const app = createApp(config, { + lease, + fenceSource: async () => { + throw new Error('stopped safely') + } + }) + + const response = await app.request(sourceFenceRequest()) + + expect(response.status).toBe(500) + expect(lease.release).not.toHaveBeenCalled() + }) +}) diff --git a/cloud/apps/relay-fence-broker/src/app.ts b/cloud/apps/relay-fence-broker/src/app.ts new file mode 100644 index 00000000000..76d90ae205c --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/app.ts @@ -0,0 +1,162 @@ +import { Hono } from 'hono' +import { z } from 'zod' +import type { RelayFenceBrokerConfig } from './config.js' +import { + GoogleStorageMutationLease, + MutationLeaseConflict +} from './mutation-lease.js' +import { + runSourceFence, + runTargetSupersession +} from './fence-operation-runner.js' + +const expectedLeaseSchema = z + .object({ + generation: z.string().regex(/^[1-9][0-9]{0,30}$/), + operationId: z.string().regex(/^[A-Za-z0-9_-]{8,128}$/), + requestDigest: z.string().regex(/^[a-f0-9]{64}$/) + }) + .strict() + +const supersessionRequestSchema = z.object({ + v: z.literal(1), + operationId: z.string().regex(/^[A-Za-z0-9_-]{8,128}$/), + fenceCommit: z.string().regex(/^[a-f0-9]{40}$/), + completedFenceRecovery: z + .object({ + attemptId: z.string().uuid(), + fenceCommit: z.string().regex(/^[a-f0-9]{40}$/), + gceOperation: z.string().min(1).max(256), + terraformStateSerial: z.number().int().nonnegative().safe(), + planObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/), + terraformStateObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/), + terraformStateObjectSha256: z.string().regex(/^[a-f0-9]{64}$/) + }) + .strict() + .optional(), + expectedLease: expectedLeaseSchema.optional(), + confirmation: z.literal('SUPERSEDE_TARGET') +}) + .strict() + .refine( + (value) => + (!value.completedFenceRecovery || Boolean(value.expectedLease)) && + (!value.expectedLease || value.expectedLease.operationId === value.operationId) + ) + +const cellIdSchema = z.string().regex(/^[a-z][a-z0-9-]{0,127}$/) +const sourceFenceRequestSchema = z + .object({ + v: z.literal(1), + operationId: z.string().regex(/^[A-Za-z0-9_-]{8,128}$/), + fenceCommit: z.string().regex(/^[a-f0-9]{40}$/), + targetCellIds: z.array(cellIdSchema).min(1).max(16), + expectedLease: expectedLeaseSchema.optional(), + confirmation: z.literal('FENCE_SOURCE') + }) + .strict() + .refine( + (value) => + new Set(value.targetCellIds).size === value.targetCellIds.length && + (!value.expectedLease || value.expectedLease.operationId === value.operationId) + ) + +type AppDependencies = { + lease?: GoogleStorageMutationLease + supersede?: ( + config: RelayFenceBrokerConfig, + recovery?: z.infer['completedFenceRecovery'] + ) => Promise + fenceSource?: ( + config: RelayFenceBrokerConfig, + targetCellIds: string[] + ) => Promise +} + +export function createApp( + config: RelayFenceBrokerConfig, + dependencies: AppDependencies = {} +): Hono { + const app = new Hono() + const lease = + dependencies.lease ?? + new GoogleStorageMutationLease( + config.stateBucket, + config.leaseObject, + config.imageCommit + ) + const supersede = dependencies.supersede ?? runTargetSupersession + const fenceSource = dependencies.fenceSource ?? runSourceFence + + app.get('/healthz', (context) => + context.json({ ok: true, fenceCommit: config.imageCommit }) + ) + app.post('/v1/supersede-target', async (context) => { + const parsed = supersessionRequestSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!parsed.success) return context.json({ error: 'invalid_request' }, 400) + if (parsed.data.fenceCommit !== config.imageCommit) { + return context.json({ error: 'fence_commit_mismatch' }, 409) + } + let acquired + try { + acquired = await lease.acquire( + parsed.data.operationId, + parsed.data, + parsed.data.expectedLease + ) + } catch (error) { + if (error instanceof MutationLeaseConflict) { + return context.json({ error: 'mutation_lease_conflict' }, 409) + } + throw error + } + await supersede(config, parsed.data.completedFenceRecovery) + await lease.release(acquired) + return context.json({ + ok: true, + operationId: parsed.data.operationId, + fenceCommit: config.imageCommit + }) + }) + app.post('/v1/fence-source', async (context) => { + const parsed = sourceFenceRequestSchema.safeParse( + await context.req.json().catch(() => null) + ) + if ( + !parsed.success || + parsed.data.targetCellIds.includes(config.sourceCellId) + ) { + return context.json({ error: 'invalid_request' }, 400) + } + if (parsed.data.fenceCommit !== config.imageCommit) { + return context.json({ error: 'fence_commit_mismatch' }, 409) + } + let acquired + try { + acquired = await lease.acquire( + parsed.data.operationId, + parsed.data, + parsed.data.expectedLease + ) + } catch (error) { + if (error instanceof MutationLeaseConflict) { + return context.json({ error: 'mutation_lease_conflict' }, 409) + } + throw error + } + await fenceSource(config, parsed.data.targetCellIds) + await lease.release(acquired) + return context.json({ + ok: true, + operationId: parsed.data.operationId, + fenceCommit: config.imageCommit + }) + }) + app.onError((error, context) => { + console.error(error) + return context.json({ error: 'broker_operation_failed' }, 500) + }) + return app +} diff --git a/cloud/apps/relay-fence-broker/src/config.ts b/cloud/apps/relay-fence-broker/src/config.ts new file mode 100644 index 00000000000..baf8340cd65 --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/config.ts @@ -0,0 +1,92 @@ +import { z } from 'zod' + +const environmentSchema = z.object({ + PORT: z.coerce.number().int().positive().max(65_535).default(8080), + ORCA_RELAY_FENCE_PROJECT: z.string().regex(/^[a-z][a-z0-9-]{4,29}$/), + ORCA_RELAY_FENCE_STATE_BUCKET: z.string().min(3).max(222), + ORCA_RELAY_FENCE_LEASE_OBJECT: z.string().min(1).max(512), + ORCA_RELAY_FENCE_DIRECTOR_ORIGIN: z.string().url(), + ORCA_RELAY_FENCE_ADMIN_AUDIENCE: z.string().url(), + ORCA_RELAY_FENCE_REQUESTER_SERVICE_ACCOUNT: z.string().email(), + ORCA_RELAY_FENCE_RUNTIME_SERVICE_ACCOUNT: z.string().email(), + ORCA_RELAY_FENCE_SOURCE_CELL_ID: z.string().regex(/^[a-z][a-z0-9-]{0,127}$/), + ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID: z + .string() + .regex(/^[a-z][a-z0-9-]{0,127}$/), + ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID: z + .string() + .regex(/^[a-z][a-z0-9-]{0,127}$/), + ORCA_RELAY_FENCE_IMAGE_COMMIT: z.string().regex(/^[a-f0-9]{40}$/), + // Resolved against the broker's working directory, which the image sets to the copied tree root. + ORCA_RELAY_FENCE_TERRAFORM_DIR: z.string().min(1).default('infra/terraform'), + ORCA_RELAY_FENCE_UNOBSERVED_CONNECTION_BOUND: z.coerce + .number() + .int() + .nonnegative() + .max(499), + ORCA_RELAY_FENCE_CONNECTION_CEILING: z.coerce + .number() + .int() + .positive() + .max(600) + .default(600) +}) + +export type RelayFenceBrokerConfig = { + port: number + project: string + stateBucket: string + leaseObject: string + directorOrigin: string + adminAudience: string + requesterServiceAccount: string + runtimeServiceAccount: string + sourceCellId: string + failedTargetCellId: string + replacementTargetCellId: string + imageCommit: string + terraformDir: string + unobservedConnectionBound: number + connectionCeiling: number +} + +export function loadConfig( + environment: NodeJS.ProcessEnv = process.env +): RelayFenceBrokerConfig { + const parsed = environmentSchema.parse(environment) + const director = new URL(parsed.ORCA_RELAY_FENCE_DIRECTOR_ORIGIN) + const audience = new URL(parsed.ORCA_RELAY_FENCE_ADMIN_AUDIENCE) + if ( + director.origin !== parsed.ORCA_RELAY_FENCE_DIRECTOR_ORIGIN || + audience.origin !== director.origin || + audience.pathname !== '/v1/admin/drain' || + audience.search || + audience.hash + ) { + throw new Error('broker director and admin audience must use the canonical drain origin') + } + const cells = new Set([ + parsed.ORCA_RELAY_FENCE_SOURCE_CELL_ID, + parsed.ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID, + parsed.ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID + ]) + if (cells.size !== 3) throw new Error('broker cells must be distinct') + return { + port: parsed.PORT, + project: parsed.ORCA_RELAY_FENCE_PROJECT, + stateBucket: parsed.ORCA_RELAY_FENCE_STATE_BUCKET, + leaseObject: parsed.ORCA_RELAY_FENCE_LEASE_OBJECT, + directorOrigin: parsed.ORCA_RELAY_FENCE_DIRECTOR_ORIGIN, + adminAudience: parsed.ORCA_RELAY_FENCE_ADMIN_AUDIENCE, + requesterServiceAccount: parsed.ORCA_RELAY_FENCE_REQUESTER_SERVICE_ACCOUNT, + runtimeServiceAccount: parsed.ORCA_RELAY_FENCE_RUNTIME_SERVICE_ACCOUNT, + sourceCellId: parsed.ORCA_RELAY_FENCE_SOURCE_CELL_ID, + failedTargetCellId: parsed.ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID, + replacementTargetCellId: parsed.ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID, + imageCommit: parsed.ORCA_RELAY_FENCE_IMAGE_COMMIT, + terraformDir: parsed.ORCA_RELAY_FENCE_TERRAFORM_DIR, + unobservedConnectionBound: + parsed.ORCA_RELAY_FENCE_UNOBSERVED_CONNECTION_BOUND, + connectionCeiling: parsed.ORCA_RELAY_FENCE_CONNECTION_CEILING + } +} diff --git a/cloud/apps/relay-fence-broker/src/fence-operation-runner.test.ts b/cloud/apps/relay-fence-broker/src/fence-operation-runner.test.ts new file mode 100644 index 00000000000..b48898977cd --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/fence-operation-runner.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest' +import type { RelayFenceBrokerConfig } from './config.js' +import { + fenceChildEnvironment, + sourceFenceArguments +} from './fence-operation-runner.js' + +const config = { + project: 'onorca-cloud', + directorOrigin: 'https://relay.onorca.dev', + adminAudience: 'https://relay.onorca.dev/v1/admin/drain', + sourceCellId: 'production-gce-c3', + runtimeServiceAccount: 'runtime@example.com', + imageCommit: 'a'.repeat(40), + terraformDir: 'infra/terraform', + unobservedConnectionBound: 10, + connectionCeiling: 600 +} as RelayFenceBrokerConfig + +describe('fence operation runner', () => { + it('passes distinct read and mutation identity tokens', () => { + expect( + fenceChildEnvironment( + config, + 'read.token.value', + 'mutation.token.value', + { PRESERVED: 'yes' } + ) + ).toMatchObject({ + PRESERVED: 'yes', + IAC_TOOL: 'terraform', + ORCA_RELAY_ADMIN_ID_TOKEN: 'read.token.value', + ORCA_RELAY_FENCE_MUTATION_ID_TOKEN: 'mutation.token.value', + ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40) + }) + }) + + it('binds a source fence to deterministic targets and the production var file', () => { + const args = sourceFenceArguments(config, '/tmp/topology.json', [ + 'production-gce-c12', + 'production-gce-c7' + ]) + + expect(args).toEqual( + expect.arrayContaining([ + '--source-cell-id', + 'production-gce-c3', + '--target-cell-ids', + 'production-gce-c12,production-gce-c7', + '--terraform-var-file', + 'environments/production.tfvars', + '--mode', + 'fence-source' + ]) + ) + }) +}) diff --git a/cloud/apps/relay-fence-broker/src/fence-operation-runner.ts b/cloud/apps/relay-fence-broker/src/fence-operation-runner.ts new file mode 100644 index 00000000000..e7fbff5a6ab --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/fence-operation-runner.ts @@ -0,0 +1,185 @@ +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import type { RelayFenceBrokerConfig } from './config.js' +import { + metadataIdentityToken, + metadataServiceAccountEmail +} from './google-metadata.js' + +const exec = promisify(execFile) +const MAX_OUTPUT_BYTES = 10 * 1024 * 1024 +type CompletedFenceRecovery = { + attemptId: string + fenceCommit: string + gceOperation: string + terraformStateSerial: number + planObjectGeneration: string + terraformStateObjectGeneration: string + terraformStateObjectSha256: string +} + +async function run(file: string, args: string[], environment?: NodeJS.ProcessEnv) { + return await exec(file, args, { + env: environment, + maxBuffer: MAX_OUTPUT_BYTES + }) +} + +export function fenceChildEnvironment( + config: RelayFenceBrokerConfig, + readToken: string, + mutationToken: string, + environment: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv { + return { + ...environment, + IAC_TOOL: 'terraform', + ORCA_RELAY_ADMIN_ID_TOKEN: readToken, + ORCA_RELAY_FENCE_MUTATION_ID_TOKEN: mutationToken, + ORCA_RELAY_FENCE_IMAGE_COMMIT: config.imageCommit + } +} + +function relayFenceArguments( + config: RelayFenceBrokerConfig, + topologyFile: string, + targetCellIds: string[] +): string[] { + return [ + 'dev/scripts/deploy-relay-gce-multi-target.mjs', + '--project', + config.project, + '--director-origin', + config.directorOrigin, + '--admin-audience', + config.adminAudience, + '--topology-file', + topologyFile, + '--source-cell-id', + config.sourceCellId, + '--target-cell-ids', + [...targetCellIds].sort().join(','), + '--unobserved-connection-bound', + String(config.unobservedConnectionBound), + '--runtime-service-account', + config.runtimeServiceAccount, + '--environment', + 'production', + '--fence-commit', + config.imageCommit, + '--terraform-dir', + config.terraformDir, + '--terraform-var-file', + 'environments/production.tfvars', + '--connection-ceiling', + String(config.connectionCeiling), + '--minimum-lease-remaining-ms', + '600000' + ] +} + +async function runRelayFenceCommand( + config: RelayFenceBrokerConfig, + argumentsForTopology: ( + topologyFile: string, + brokerServiceAccount: string + ) => string[] +): Promise { + const directory = await mkdtemp(join(tmpdir(), 'orca-relay-fence-operation-')) + const topologyFile = join(directory, 'topology.json') + try { + await run('terraform', [ + `-chdir=${config.terraformDir}`, + 'init', + '-input=false', + '-backend-config=backend/production.hcl' + ]) + const topology = await run('terraform', [ + `-chdir=${config.terraformDir}`, + 'output', + '-json', + 'relay_gce_cell_deployments' + ]) + await writeFile(topologyFile, topology.stdout, { mode: 0o600 }) + const brokerToken = await metadataIdentityToken(config.adminAudience) + const brokerServiceAccount = await metadataServiceAccountEmail() + const environment = fenceChildEnvironment( + config, + brokerToken, + brokerToken + ) + await run( + 'node', + argumentsForTopology(topologyFile, brokerServiceAccount), + environment + ) + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +export function sourceFenceArguments( + config: RelayFenceBrokerConfig, + topologyFile: string, + targetCellIds: string[] +): string[] { + return [ + ...relayFenceArguments(config, topologyFile, targetCellIds), + '--mode', + 'fence-source' + ] +} + +export async function runSourceFence( + config: RelayFenceBrokerConfig, + targetCellIds: string[] +): Promise { + await runRelayFenceCommand(config, (topologyFile) => + sourceFenceArguments(config, topologyFile, targetCellIds) + ) +} + +export async function runTargetSupersession( + config: RelayFenceBrokerConfig, + recovery?: CompletedFenceRecovery +): Promise { + await runRelayFenceCommand(config, (topologyFile, brokerServiceAccount) => { + const targets = [ + config.failedTargetCellId, + config.replacementTargetCellId + ] + const args = [ + ...relayFenceArguments(config, topologyFile, targets), + '--failed-target-cell-id', + config.failedTargetCellId, + '--replacement-target-cell-id', + config.replacementTargetCellId, + '--mode', + 'supersede-target' + ] + if (recovery) { + args.push( + '--completed-fence-attempt-id', + recovery.attemptId, + '--completed-fence-commit', + recovery.fenceCommit, + '--completed-fence-operation', + recovery.gceOperation, + '--completed-fence-state-serial', + String(recovery.terraformStateSerial), + '--completed-fence-plan-generation', + recovery.planObjectGeneration, + '--completed-fence-state-generation', + recovery.terraformStateObjectGeneration, + '--completed-fence-state-sha256', + recovery.terraformStateObjectSha256, + '--fence-broker-service-account', + brokerServiceAccount + ) + } + return args + }) +} diff --git a/cloud/apps/relay-fence-broker/src/google-metadata.ts b/cloud/apps/relay-fence-broker/src/google-metadata.ts new file mode 100644 index 00000000000..ffae11e392c --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/google-metadata.ts @@ -0,0 +1,44 @@ +const METADATA_ROOT = + 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default' + +async function metadataText(path: string, fetcher: typeof fetch): Promise { + const response = await fetcher(`${METADATA_ROOT}/${path}`, { + headers: { 'Metadata-Flavor': 'Google' } + }) + if (!response.ok) throw new Error(`metadata request failed: ${response.status}`) + return await response.text() +} + +export async function metadataAccessToken(fetcher: typeof fetch = fetch): Promise { + const body = JSON.parse(await metadataText('token', fetcher)) as { + access_token?: unknown + } + if (typeof body.access_token !== 'string' || body.access_token.length < 20) { + throw new Error('metadata access token is missing') + } + return body.access_token +} + +export async function metadataServiceAccountEmail( + fetcher: typeof fetch = fetch +): Promise { + const email = (await metadataText('email', fetcher)).trim() + if (!/^[^@\s]+@[^@\s]+\.gserviceaccount\.com$/.test(email)) { + throw new Error('metadata service account email is invalid') + } + return email +} + +export async function metadataIdentityToken( + audience: string, + fetcher: typeof fetch = fetch +): Promise { + const token = await metadataText( + `identity?audience=${encodeURIComponent(audience)}&format=full`, + fetcher + ) + if (!/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) { + throw new Error('metadata identity token is invalid') + } + return token +} diff --git a/cloud/apps/relay-fence-broker/src/index.ts b/cloud/apps/relay-fence-broker/src/index.ts new file mode 100644 index 00000000000..cd237b964f1 --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/index.ts @@ -0,0 +1,10 @@ +import { serve } from '@hono/node-server' +import { createApp } from './app.js' +import { loadConfig } from './config.js' + +const config = loadConfig() +const app = createApp(config) + +serve({ fetch: app.fetch, port: config.port }, (info) => { + console.log(`[relay-fence-broker] listening on port ${info.port}`) +}) diff --git a/cloud/apps/relay-fence-broker/src/mutation-lease.test.ts b/cloud/apps/relay-fence-broker/src/mutation-lease.test.ts new file mode 100644 index 00000000000..6006e207ce7 --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/mutation-lease.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it, vi } from 'vitest' +import { + GoogleStorageMutationLease, + MutationLeaseConflict +} from './mutation-lease.js' + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' } + }) +} + +const accessToken = () => json({ access_token: 'a'.repeat(40) }) +const request = { + v: 1, + operationId: 'c11-to-c12-forward', + fenceCommit: 'a'.repeat(40), + confirmation: 'SUPERSEDE_TARGET' +} + +describe('GoogleStorageMutationLease', () => { + it('creates and conditionally releases a new lease', async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce(accessToken()) + .mockResolvedValueOnce(new Response(null, { status: 404 })) + .mockResolvedValueOnce(json({ generation: '7' })) + .mockResolvedValueOnce(accessToken()) + .mockResolvedValueOnce(new Response(null, { status: 204 })) + const leaseStore = new GoogleStorageMutationLease( + 'state-bucket', + 'fence/production.lock', + 'a'.repeat(40), + fetcher, + () => 1_000 + ) + + const lease = await leaseStore.acquire(request.operationId, request) + await leaseStore.release(lease) + + expect(lease.generation).toBe('7') + expect(fetcher.mock.calls[2]?.[0]).toContain('ifGenerationMatch=0') + expect(fetcher.mock.calls[4]?.[0]).toContain('ifGenerationMatch=7') + }) + + it('rejects a different request while the durable lease is live', async () => { + const existing = { + operationId: 'other-operation', + requestDigest: 'b'.repeat(64), + imageCommit: 'a'.repeat(40), + acquiredAt: 500, + expiresAt: 2_000 + } + const fetcher = vi + .fn() + .mockResolvedValueOnce(accessToken()) + .mockResolvedValueOnce(json({ generation: '7' })) + .mockResolvedValueOnce(json(existing)) + const leaseStore = new GoogleStorageMutationLease( + 'state-bucket', + 'fence/production.lock', + 'a'.repeat(40), + fetcher, + () => 1_000 + ) + + await expect( + leaseStore.acquire(request.operationId, request) + ).rejects.toBeInstanceOf(MutationLeaseConflict) + expect(fetcher).toHaveBeenCalledTimes(3) + }) + + it('takes over an expired lease with an exact generation precondition', async () => { + const existing = { + operationId: 'other-operation', + requestDigest: 'b'.repeat(64), + imageCommit: 'a'.repeat(40), + acquiredAt: 500, + expiresAt: 999 + } + const fetcher = vi + .fn() + .mockResolvedValueOnce(accessToken()) + .mockResolvedValueOnce(json({ generation: '7' })) + .mockResolvedValueOnce(json(existing)) + .mockResolvedValueOnce(json({ generation: '8' })) + const leaseStore = new GoogleStorageMutationLease( + 'state-bucket', + 'fence/production.lock', + 'a'.repeat(40), + fetcher, + () => 1_000 + ) + + const lease = await leaseStore.acquire(request.operationId, request) + + expect(lease.generation).toBe('8') + expect(fetcher.mock.calls[3]?.[0]).toContain('ifGenerationMatch=7') + }) + + it('conditionally replaces the exact authorized live lease', async () => { + const existing = { + operationId: request.operationId, + requestDigest: 'b'.repeat(64), + imageCommit: 'b'.repeat(40), + acquiredAt: 500, + expiresAt: 2_000 + } + const fetcher = vi + .fn() + .mockResolvedValueOnce(accessToken()) + .mockResolvedValueOnce(json({ generation: '7' })) + .mockResolvedValueOnce(json(existing)) + .mockResolvedValueOnce(json({ generation: '8' })) + const leaseStore = new GoogleStorageMutationLease( + 'state-bucket', + 'fence/production.lock', + 'a'.repeat(40), + fetcher, + () => 1_000 + ) + + const lease = await leaseStore.acquire(request.operationId, request, { + generation: '7', + operationId: existing.operationId, + requestDigest: existing.requestDigest + }) + + expect(lease.generation).toBe('8') + expect(fetcher.mock.calls[3]?.[0]).toContain('ifGenerationMatch=7') + }) + + it('rejects a live-lease takeover when any expected field differs', async () => { + const existing = { + operationId: request.operationId, + requestDigest: 'b'.repeat(64), + imageCommit: 'b'.repeat(40), + acquiredAt: 500, + expiresAt: 2_000 + } + const fetcher = vi + .fn() + .mockResolvedValueOnce(accessToken()) + .mockResolvedValueOnce(json({ generation: '7' })) + .mockResolvedValueOnce(json(existing)) + const leaseStore = new GoogleStorageMutationLease( + 'state-bucket', + 'fence/production.lock', + 'a'.repeat(40), + fetcher, + () => 1_000 + ) + + await expect( + leaseStore.acquire(request.operationId, request, { + generation: '8', + operationId: existing.operationId, + requestDigest: existing.requestDigest + }) + ).rejects.toBeInstanceOf(MutationLeaseConflict) + expect(fetcher).toHaveBeenCalledTimes(3) + }) +}) diff --git a/cloud/apps/relay-fence-broker/src/mutation-lease.ts b/cloud/apps/relay-fence-broker/src/mutation-lease.ts new file mode 100644 index 00000000000..38bd4a1cdfe --- /dev/null +++ b/cloud/apps/relay-fence-broker/src/mutation-lease.ts @@ -0,0 +1,152 @@ +import { createHash } from 'node:crypto' +import { metadataAccessToken } from './google-metadata.js' + +const LEASE_TTL_MS = 35 * 60 * 1_000 + +type LeaseRecord = { + operationId: string + requestDigest: string + imageCommit: string + acquiredAt: number + expiresAt: number +} + +type ObjectMetadata = { + generation: string +} + +export class MutationLeaseConflict extends Error {} + +export type MutationLease = { + generation: string + record: LeaseRecord +} + +export type ExpectedMutationLease = { + generation: string + operationId: string + requestDigest: string +} + +export class GoogleStorageMutationLease { + constructor( + private readonly bucket: string, + private readonly objectName: string, + private readonly imageCommit: string, + private readonly fetcher: typeof fetch = fetch, + private readonly now: () => number = Date.now + ) {} + + async acquire( + operationId: string, + request: unknown, + expectedExisting?: ExpectedMutationLease + ): Promise { + const token = await metadataAccessToken(this.fetcher) + const existing = await this.read(token) + const requestDigest = createHash('sha256') + .update(JSON.stringify(request)) + .digest('hex') + const exactTakeover = + existing && + expectedExisting?.generation === existing.metadata.generation && + expectedExisting.operationId === existing.record.operationId && + expectedExisting.requestDigest === existing.record.requestDigest + if ( + (!existing && expectedExisting) || + (existing && + existing.record.requestDigest !== requestDigest && + existing.record.expiresAt > this.now() && + !exactTakeover) + ) { + throw new MutationLeaseConflict('another relay mutation owns the durable lease') + } + const acquiredAt = this.now() + const record: LeaseRecord = { + operationId, + requestDigest, + imageCommit: this.imageCommit, + acquiredAt, + expiresAt: acquiredAt + LEASE_TTL_MS + } + const generation = existing?.metadata.generation ?? '0' + const response = await this.fetcher( + `${this.uploadUrl()}&ifGenerationMatch=${encodeURIComponent(generation)}`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify(record) + } + ) + if (response.status === 412) { + throw new MutationLeaseConflict('relay mutation lease changed concurrently') + } + if (!response.ok) throw new Error(`mutation lease acquisition failed: ${response.status}`) + const metadata = (await response.json()) as Partial + if (!/^[1-9][0-9]{0,30}$/.test(metadata.generation ?? '')) { + throw new Error('mutation lease has no valid generation') + } + return { generation: metadata.generation!, record } + } + + async release(lease: MutationLease): Promise { + const token = await metadataAccessToken(this.fetcher) + const response = await this.fetcher( + `${this.metadataUrl()}?ifGenerationMatch=${encodeURIComponent(lease.generation)}`, + { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + } + ) + if (!response.ok && response.status !== 404) { + throw new Error(`mutation lease release failed: ${response.status}`) + } + } + + private async read( + token: string + ): Promise<{ metadata: ObjectMetadata; record: LeaseRecord } | null> { + const metadataResponse = await this.fetcher(this.metadataUrl(), { + headers: { Authorization: `Bearer ${token}` } + }) + if (metadataResponse.status === 404) return null + if (!metadataResponse.ok) { + throw new Error(`mutation lease inspection failed: ${metadataResponse.status}`) + } + const metadata = (await metadataResponse.json()) as Partial + if (!/^[1-9][0-9]{0,30}$/.test(metadata.generation ?? '')) { + throw new Error('existing mutation lease has no valid generation') + } + const bodyResponse = await this.fetcher(`${this.metadataUrl()}?alt=media`, { + headers: { Authorization: `Bearer ${token}` } + }) + if (!bodyResponse.ok) { + throw new Error(`mutation lease body read failed: ${bodyResponse.status}`) + } + const record = (await bodyResponse.json()) as Partial + if ( + typeof record.operationId !== 'string' || + !/^[a-f0-9]{64}$/.test(record.requestDigest ?? '') || + !/^[a-f0-9]{40}$/.test(record.imageCommit ?? '') || + !Number.isSafeInteger(record.acquiredAt) || + !Number.isSafeInteger(record.expiresAt) + ) { + throw new Error('existing mutation lease is invalid') + } + return { + metadata: { generation: metadata.generation! }, + record: record as LeaseRecord + } + } + + private metadataUrl(): string { + return `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(this.bucket)}/o/${encodeURIComponent(this.objectName)}` + } + + private uploadUrl(): string { + return `https://storage.googleapis.com/upload/storage/v1/b/${encodeURIComponent(this.bucket)}/o?uploadType=media&name=${encodeURIComponent(this.objectName)}` + } +} diff --git a/cloud/apps/relay-fence-broker/tsconfig.build.json b/cloud/apps/relay-fence-broker/tsconfig.build.json new file mode 100644 index 00000000000..489ddfd34d6 --- /dev/null +++ b/cloud/apps/relay-fence-broker/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "noEmit": false, + "outDir": "dist", + "rootDir": "src" + }, + "exclude": ["src/**/*.test.ts"] +} diff --git a/cloud/apps/relay-fence-broker/tsconfig.json b/cloud/apps/relay-fence-broker/tsconfig.json new file mode 100644 index 00000000000..a552e34dbe9 --- /dev/null +++ b/cloud/apps/relay-fence-broker/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true }, + "include": ["src/**/*.ts"] +} diff --git a/cloud/apps/relay-ops/README.md b/cloud/apps/relay-ops/README.md new file mode 100644 index 00000000000..7dd16307dc0 --- /dev/null +++ b/cloud/apps/relay-ops/README.md @@ -0,0 +1,66 @@ +# Orca Relay Operations + +A private, aggregate dashboard for the Orca Relay control and data planes. It reads local `gcloud` and `gh` credentials on the server; credentials and per-user Relay state never enter the browser. One cached `gcloud auth print-access-token` refresh feeds concurrent read-only Google APIs so the collector does not stampede the local credential store. + +## Run locally + +Prerequisites: + +- Node 24 and pnpm 10 +- `gcloud` authenticated for `onorca-cloud` and `onorca-cloud-staging` +- `gh` authenticated with read access to `stablyai/orca-cloud` + +From the repository root: + +```sh +pnpm install +pnpm ops:relay +``` + +Open . The server binds only to loopback and refreshes aggregate data every minute. Production and staging are read-only by default. + +The cost panel is a labeled planning estimate. There is currently no Cloud Billing export in either project, so the dashboard cannot claim exact billed spend. GCP Billing remains authoritative. + +## Share through Tailscale + +Keep the dashboard bound to loopback and let Tailscale provide identity, TLS, and tailnet ACL enforcement: + +```sh +tailscale serve --bg http://127.0.0.1:2455 +tailscale serve status +``` + +Share the HTTPS URL printed by `tailscale serve status` with the team. Limit access to the intended operator group in the tailnet ACL. Do not use a public funnel. Stop sharing with: + +```sh +tailscale serve reset +``` + +For a persistent host, run `pnpm --filter @orca-cloud/relay-ops build` and supervise `pnpm --filter @orca-cloud/relay-ops start` with the host's normal process manager. The process needs the same non-interactive `gcloud` and `gh` identities. + +## Optional staging controls + +Controls are intentionally local-only and disabled unless explicitly enabled: + +```sh +RELAY_OPS_ENABLE_STAGING_CONTROLS=1 pnpm ops:relay +``` + +Even in this mode the service never changes GCP directly. It dispatches `.github/workflows/power-relay-staging.yml`, preserves the workflow's typed `WAKE_STAGING` / `SLEEP_STAGING` confirmation, and always wakes only configured-admission cells. Requests require the loopback origin and a per-process CSRF token, so controls stay unavailable through the Tailscale view. + +## Data and security boundaries + +- Browser payloads contain aggregate Monitoring points, resource health, immutable image digests, alert-policy metadata, and workflow metadata. +- Account IDs, host IDs, device IDs, pairing state, assignment rows, bearer tokens, service-account tokens, startup scripts, secret values, and individual Relay-admin state are excluded. +- Sleeping staging is inventory-only. Viewing it does not probe or cold-start Cloud Run services and cannot resize empty MIGs. +- Partial GCP or GitHub failures degrade the affected panel and produce a sanitized warning. +- Missing cell inventory renders as `Unknown`, never `Sleeping`. After one successful read, transient credential or collector failures retain the last good snapshot and mark it stale. +- Responses use `no-store`, a restrictive CSP, frame denial, and no-referrer headers. + +## Verification + +```sh +pnpm --filter @orca-cloud/relay-ops test +pnpm --filter @orca-cloud/relay-ops typecheck +pnpm --filter @orca-cloud/relay-ops build +``` diff --git a/cloud/apps/relay-ops/package.json b/cloud/apps/relay-ops/package.json new file mode 100644 index 00000000000..da4f73f8672 --- /dev/null +++ b/cloud/apps/relay-ops/package.json @@ -0,0 +1,29 @@ +{ + "name": "@orca-cloud/relay-ops", + "private": true, + "version": "0.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "pnpm clean && tsc -p tsconfig.build.json && node -e \"require('fs').cpSync('public','dist/public',{recursive:true})\"", + "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", + "dev": "tsx watch src/index.ts", + "incident:monitor": "tsx src/incident-monitor-cli.ts", + "incident:preflight": "tsx src/incident-live-preflight-cli.ts", + "lint": "tsc -p tsconfig.json --noEmit", + "start": "node dist/index.js", + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "@hono/node-server": "^1.19.14", + "hono": "^4.12.27", + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "tsx": "^4.21.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/apps/relay-ops/public/app.js b/cloud/apps/relay-ops/public/app.js new file mode 100644 index 00000000000..83c072662fb --- /dev/null +++ b/cloud/apps/relay-ops/public/app.js @@ -0,0 +1,273 @@ +const state = { environment: 'production', window: 360, snapshot: null, config: null, loading: false } + +const $ = (selector) => document.querySelector(selector) +const all = (selector) => [...document.querySelectorAll(selector)] +const escapeHtml = (value) => String(value).replace(/[&<>'"]/g, (character) => ({ + '&': '&', '<': '<', '>': '>', "'": ''', '"': '"' +})[character]) + +function formatNumber(value, maximumFractionDigits = 0) { + if (value === null || value === undefined) return '—' + return new Intl.NumberFormat('en-US', { notation: value >= 10_000 ? 'compact' : 'standard', maximumFractionDigits }).format(value) +} + +function formatBytes(value) { + if (value === null || value === undefined) return '—' + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let amount = value + let unit = 0 + while (Math.abs(amount) >= 1000 && unit < units.length - 1) { amount /= 1000; unit += 1 } + return `${formatNumber(amount, amount < 10 ? 1 : 0)} ${units[unit]}` +} + +function formatMetric(metric, value) { + if (value === null || value === undefined) return '—' + if (metric.unit === 'bytes') return formatBytes(value) + if (metric.unit === 'milliseconds') return `${formatNumber(value, 1)} ms` + return formatNumber(value, value < 10 ? 1 : 0) +} + +function timeAgo(value) { + const seconds = Math.max(0, Math.round((Date.now() - Date.parse(value)) / 1000)) + if (seconds < 60) return `${seconds}s ago` + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago` + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago` + return `${Math.floor(seconds / 86400)}d ago` +} + +function shortImage(value) { + const digest = value?.match(/sha256:([a-f0-9]{64})/)?.[1] + if (digest) return digest.slice(0, 10) + const tag = value?.split(':').at(-1) + return tag?.slice(0, 16) ?? '—' +} + +function badge(label, healthy) { + return `${escapeHtml(label)}` +} + +function renderSummary(snapshot) { + const { summary } = snapshot + const utilization = summary.poweredCapacity === null + ? null + : summary.poweredCapacity > 0 + ? summary.observedConnections / summary.poweredCapacity * 100 + : 0 + const items = [ + ['Observed connections', formatNumber(summary.observedConnections, 1), 'Latest 1-minute aggregate mean'], + ['Active relay sessions', formatNumber(summary.observedSplices, 1), `${formatNumber(summary.observedControls, 1)} desktop-control mean`], + ['Healthy cells', `${summary.activeCells ?? '—'} / ${summary.totalCells}`, summary.poweredCapacity === null ? 'Cell inventory unavailable' : `${formatNumber(summary.poweredCapacity)} powered request units`], + ['Capacity signal', utilization === null ? '—' : `${formatNumber(utilization, 2)}%`, `${formatNumber(summary.configuredCapacity)} configured admission units`] + ] + $('#summary').innerHTML = items.map(([label, value, caption]) => ` +
+

${escapeHtml(label)}

+
${escapeHtml(value)}
+
${escapeHtml(caption)}
+
`).join('') +} + +function cellState(cell) { + if (cell.targetSize === null) return ['Unknown', null] + if (cell.targetSize === 0) return ['Sleeping', null] + if (cell.backendHealth === 'healthy' && cell.endpoint.ready) return ['Healthy', true] + if (cell.stable && cell.backendHealth === 'empty') return ['Starting', null] + return ['Attention', false] +} + +function renderTopology(snapshot) { + const director = snapshot.resources.director + const sleeping = snapshot.resources.cells.every((cell) => cell.targetSize === 0) + && snapshot.resources.sql?.activationPolicy === 'NEVER' + const directorHealthy = director?.ready && snapshot.resources.directorEndpoint.health + $('#topology-meta').textContent = `${snapshot.environment.project} · ${snapshot.environment.region}` + const cellNodes = snapshot.resources.cells.map((cell) => { + const [label, healthy] = cellState(cell) + return `
+ ${escapeHtml(cell.hostname.toUpperCase())} ${badge(label, healthy)} + ${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)} · ${formatNumber(cell.capacityRequests)} units +
` + }).join('') + $('#topology').innerHTML = ` +
Desktop + phoneEncrypted Relay traffic
+ +
Director ${badge(sleeping ? 'Sleeping' : directorHealthy ? 'Ready' : 'Attention', sleeping ? null : Boolean(directorHealthy))}${escapeHtml(snapshot.environment.directorOrigin)}
+ +
${cellNodes}
` +} + +function chartPaths(points, width = 400, height = 112) { + if (points.length === 0) return null + const values = points.map((point) => point.value) + const minimum = Math.min(0, ...values) + const maximum = Math.max(...values) + const spread = maximum - minimum || 1 + const coordinates = points.map((point, index) => { + const x = points.length === 1 ? width : index / (points.length - 1) * width + const y = height - ((point.value - minimum) / spread * (height - 10) + 5) + return [x, y] + }) + const line = coordinates.map(([x, y], index) => `${index === 0 ? 'M' : 'L'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ') + const area = `${line} L${width},${height} L0,${height} Z` + return { line, area } +} + +function renderCharts(snapshot) { + const names = ['controls', 'splices', 'assignment_5xx', 'postgres_retries', 'auth_failures', 'event_loop_ms_p99'] + $('#charts').innerHTML = names.map((name) => { + const metric = snapshot.monitoring.metrics[name] + const paths = chartPaths(metric.points) + const graph = paths ? ` + + + ` : '
No samples in this window
' + return `

${escapeHtml(metric.label)}

${escapeHtml(formatMetric(metric, metric.latest))}
${escapeHtml(metric.unit)}
${graph}
` + }).join('') +} + +function renderCells(snapshot) { + const controlsByCell = snapshot.monitoring.metrics.controls.latestByCell + const splicesByCell = snapshot.monitoring.metrics.splices.latestByCell + $('#cells').innerHTML = snapshot.resources.cells.map((cell) => { + const [label, healthy] = cellState(cell) + const observed = (controlsByCell[cell.cellId] ?? 0) + (splicesByCell[cell.cellId] ?? 0) + return ` + ${escapeHtml(cell.hostname.toUpperCase())}
${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)}
+ ${badge(label, healthy)}
MIG ${cell.runningInstances ?? '—'}/${cell.targetSize ?? '—'}
+ ${formatNumber(observed, 1)}
1-minute mean
+ ${formatNumber(cell.capacityRequests)}
DB pool ${formatNumber(cell.databasePoolMax)} · ${cell.configuredAdmission ? 'configured' : 'candidate only'}
+ ${escapeHtml(shortImage(cell.imageDigest))} + ` + }).join('') +} + +function serviceRow(name, healthy, detail, suffix = '') { + return `
${escapeHtml(name)}
${escapeHtml(detail)}
${badge(suffix || (healthy ? 'Ready' : 'Attention'), healthy)}
` +} + +function renderServices(snapshot) { + const { resources } = snapshot + const sleeping = resources.cells.every((cell) => cell.targetSize === 0) + && resources.sql?.activationPolicy === 'NEVER' + const certDays = resources.certificate?.expireTime + ? Math.floor((Date.parse(resources.certificate.expireTime) - Date.now()) / 86400000) + : null + $('#services').innerHTML = [ + serviceRow('Director', sleeping ? null : Boolean(resources.director?.ready && resources.directorEndpoint.health), `${resources.director?.revision ?? 'Revision unavailable'} · ${shortImage(resources.director?.image)}`, sleeping ? 'Sleeping' : ''), + serviceRow('Authentication', sleeping ? null : Boolean(resources.auth?.ready && resources.authEndpoint.health), `${resources.auth?.revision ?? 'Revision unavailable'} · ${shortImage(resources.auth?.image)}`, sleeping ? 'Sleeping' : ''), + serviceRow('Cloud SQL', resources.sql?.state === 'RUNNABLE' || resources.sql?.activationPolicy === 'NEVER', `${resources.sql?.tier ?? 'unknown'} · ${resources.sql?.activationPolicy ?? 'unknown'}`, resources.sql?.state ?? 'Unknown'), + serviceRow('Wildcard TLS', resources.certificate?.state === 'ACTIVE', resources.certificate?.domains.join(', ') ?? 'Certificate unavailable', certDays === null ? 'Unknown' : `${certDays}d`) + ].join('') +} + +function renderAlerts(snapshot) { + const policies = snapshot.monitoring.alertPolicies + $('#alerts').innerHTML = policies.length ? policies.map((policy) => `
${escapeHtml(policy.displayName.replace('Orca Relay: ', ''))}
Cloud Monitoring policy
${badge(policy.enabled ? 'Enabled' : 'Disabled', policy.enabled)}
`).join('') : '

No Relay alert policies returned.

' +} + +function renderWorkflows(snapshot) { + $('#workflows').innerHTML = snapshot.workflows.length ? snapshot.workflows.slice(0, 6).map((run) => { + const healthy = run.conclusion === 'success' + const stateLabel = run.status === 'completed' ? (run.conclusion ?? 'completed') : run.status + return `
${escapeHtml(run.name)}
${escapeHtml(run.headSha)} · ${timeAgo(run.updatedAt)}
${badge(stateLabel, healthy)}` + }).join('') : '

Workflow history unavailable.

' +} + +function renderCost(snapshot) { + const cost = snapshot.cost + $('#cost').innerHTML = `
$${formatNumber(cost.monthlyUsd)}/ month
+

Modeled range $${formatNumber(cost.rangeUsd[0])}–$${formatNumber(cost.rangeUsd[1])}. Not billed cost.

+
${cost.lines.map((line) => `
${escapeHtml(line.label)}$${formatNumber(line.monthlyUsd, 2)}
`).join('')}
+

Exact billing: ${escapeHtml(cost.actualBilling.reason)}
${escapeHtml(cost.caveats[1])}

` +} + +function renderWarnings(snapshot) { + const warning = $('#warnings') + warning.classList.toggle('hidden', snapshot.warnings.length === 0) + warning.textContent = snapshot.warnings.length ? `Partial data: ${snapshot.warnings.join(' ')}` : '' +} + +function render(snapshot) { + state.snapshot = snapshot + $('#environment-label').textContent = `${snapshot.environment.label} · ${snapshot.environment.region}` + $('#freshness').textContent = snapshot.stale + ? `Last good update ${timeAgo(snapshot.generatedAt)} · refresh degraded` + : `Updated ${timeAgo(snapshot.generatedAt)}` + $('#freshness-dot').className = snapshot.stale ? 'status-dot neutral' : 'status-dot healthy' + renderWarnings(snapshot) + renderSummary(snapshot) + renderTopology(snapshot) + renderCharts(snapshot) + renderCells(snapshot) + renderServices(snapshot) + renderAlerts(snapshot) + renderWorkflows(snapshot) + renderCost(snapshot) + $('#power-panel').classList.toggle('hidden', !(state.environment === 'staging' && state.config?.stagingControlsEnabled)) + $('#compute-link').href = snapshot.environment.consoleLinks.compute + $('#alert-link').href = snapshot.environment.consoleLinks.alerts +} + +async function loadSnapshot() { + if (state.loading) return + state.loading = true + $('#refresh').disabled = true + $('#error').classList.add('hidden') + $('#freshness-dot').className = 'status-dot neutral' + $('#freshness').textContent = 'Refreshing current state…' + try { + const response = await fetch(`/api/snapshot?environment=${state.environment}&window=${state.window}`) + const body = await response.json() + if (!response.ok) throw new Error(body.error ?? 'Snapshot failed') + render(body) + } catch (error) { + $('#freshness-dot').className = 'status-dot unhealthy' + $('#freshness').textContent = 'Refresh failed' + $('#error').textContent = error instanceof Error ? error.message : 'Relay operations data is unavailable.' + $('#error').classList.remove('hidden') + } finally { + state.loading = false + $('#refresh').disabled = false + } +} + +async function dispatchPower(mode) { + const confirmation = mode === 'wake' ? 'WAKE_STAGING' : mode === 'sleep' ? 'SLEEP_STAGING' : '' + if (confirmation) { + const entered = window.prompt(`Type ${confirmation} to dispatch the guarded workflow.`) ?? '' + if (entered !== confirmation) return + } + all('[data-power]').forEach((button) => { button.disabled = true }) + $('#power-result').textContent = 'Dispatching…' + try { + const response = await fetch('/api/staging/power', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-csrf-token': state.config.csrfToken }, + body: JSON.stringify({ mode, confirmation }) + }) + const body = await response.json() + if (!response.ok) throw new Error(body.error) + $('#power-result').textContent = 'Workflow accepted. Refresh after it completes.' + } catch (error) { + $('#power-result').textContent = error instanceof Error ? error.message : 'Dispatch failed.' + } finally { + all('[data-power]').forEach((button) => { button.disabled = false }) + } +} + +all('[data-environment]').forEach((button) => button.addEventListener('click', () => { + state.environment = button.dataset.environment + all('[data-environment]').forEach((candidate) => candidate.setAttribute('aria-pressed', String(candidate === button))) + loadSnapshot() +})) +$('#window').addEventListener('change', (event) => { state.window = Number(event.target.value); loadSnapshot() }) +$('#refresh').addEventListener('click', loadSnapshot) +all('[data-power]').forEach((button) => button.addEventListener('click', () => dispatchPower(button.dataset.power))) + +async function start() { + try { state.config = await fetch('/api/config').then((response) => response.json()) } catch { state.config = {} } + await loadSnapshot() + window.setInterval(loadSnapshot, 60_000) +} + +start() diff --git a/cloud/apps/relay-ops/public/index.html b/cloud/apps/relay-ops/public/index.html new file mode 100644 index 00000000000..c49c3bee579 --- /dev/null +++ b/cloud/apps/relay-ops/public/index.html @@ -0,0 +1,115 @@ + + + + + + + Orca Relay Operations + + + + + +
+
+ +
+ Orca Relay + Operations +
+
+
+
+ + +
+ + +
+
+ +
+
+
+

Production · us-central1

+

Relay data plane

+

Private, aggregate operations view. No account, host, device, or pairing identifiers.

+
+
+ + Loading current state… +
+
+ + + + +
+
+
+
+
+
+ +
+
+

Topology

Request path

+ +
+
+
+ +

Traffic

Signals in selected window

+
+ +
+
+

Compute

Relay cells

Open in GCP ↗
+

Configured admission is shown below. Live director admission and heartbeat state stays unavailable because this dashboard has no Relay-admin identity.

+
+ + +
CellStateObservedCapacityImage
+
+
+

Control plane

Services

+
+
+
+ +
+ +
+

Delivery

Recent workflows

+
+
+
+

Planning

Monthly run-rate

+
+
+
+ + +
+
Orca Relay Operations · Server-side credentials · Read-only by default
+ + diff --git a/cloud/apps/relay-ops/public/styles.css b/cloud/apps/relay-ops/public/styles.css new file mode 100644 index 00000000000..17a974c3d9a --- /dev/null +++ b/cloud/apps/relay-ops/public/styles.css @@ -0,0 +1,165 @@ +:root { + --radius: 10px; + --background: #fff; + --foreground: #0a0a0a; + --card: #fff; + --primary: #171717; + --primary-foreground: #fafafa; + --secondary: #f5f5f5; + --muted: #f5f5f5; + --muted-foreground: #737373; + --accent: #f5f5f5; + --border: #e5e5e5; + --ring: #a1a1a1; + --destructive: #e40014; + --success: #15803d; + --warning: #895503; + --font-mono: 'SF Mono', SFMono-Regular, ui-monospace, Menlo, Consolas, monospace; +} +@media (prefers-color-scheme: dark) { + :root { + --background: #0a0a0a; + --foreground: #fafafa; + --card: #171717; + --primary: #e5e5e5; + --primary-foreground: #171717; + --secondary: #262626; + --muted: #262626; + --muted-foreground: #a1a1a1; + --accent: #262626; + --border: rgb(255 255 255 / 0.07); + --ring: #737373; + --destructive: #ff6568; + --success: #4ade80; + --warning: #fbbf24; + } +} +* { box-sizing: border-box; } +body { + margin: 0; + background: var(--background); + color: var(--foreground); + font-family: 'Geist', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; + font-size: 14px; + letter-spacing: .01em; +} +button, select { font: inherit; } +button:focus-visible, select:focus-visible { outline: 2px solid var(--ring); outline-offset: 2px; } +.topbar { + height: 64px; + border-bottom: 1px solid var(--border); + padding: 0 max(24px, calc((100vw - 1440px) / 2)); + display: flex; + align-items: center; + justify-content: space-between; + position: sticky; + top: 0; + background: color-mix(in srgb, var(--background) 92%, transparent); + backdrop-filter: blur(16px); + z-index: 10; +} +.brand { display: flex; align-items: center; gap: 10px; } +.brand-mark { + width: 30px; height: 30px; border-radius: 9px; background: var(--primary); + color: var(--primary-foreground); display: grid; place-items: center; font-weight: 700; +} +.brand div { display: flex; align-items: baseline; gap: 7px; } +.brand span:last-child { color: var(--muted-foreground); font-size: 12px; } +.toolbar { display: flex; align-items: center; gap: 8px; } +.segmented { display: flex; padding: 3px; gap: 2px; border-radius: 8px; background: var(--secondary); } +.segmented button { border: 0; background: transparent; color: var(--muted-foreground); padding: 5px 10px; border-radius: 6px; cursor: pointer; } +.segmented button[aria-pressed="true"] { background: var(--card); color: var(--foreground); box-shadow: 0 1px 2px rgb(0 0 0 / .08); } +select, .button { height: 32px; border: 1px solid var(--border); border-radius: 8px; background: var(--card); color: var(--foreground); padding: 0 10px; } +.button { cursor: pointer; font-weight: 500; } +.button:hover { background: var(--accent); } +.button:disabled { opacity: .5; cursor: wait; } +main { max-width: 1440px; margin: 0 auto; padding: 42px 24px 72px; } +.page-heading { display: flex; justify-content: space-between; align-items: end; gap: 24px; margin-bottom: 28px; } +h1, h2, p { margin: 0; } +h1 { font-size: 28px; line-height: 1.2; letter-spacing: -.025em; margin-top: 5px; } +h2 { font-size: 16px; line-height: 1.25; letter-spacing: -.01em; margin-top: 3px; } +.lede { color: var(--muted-foreground); margin-top: 8px; max-width: 680px; } +.eyebrow { color: var(--muted-foreground); font-size: 11px; line-height: 1; font-weight: 600; text-transform: uppercase; letter-spacing: .05em; } +.freshness { color: var(--muted-foreground); display: flex; align-items: center; gap: 8px; font-size: 12px; white-space: nowrap; } +.status-dot { width: 7px; height: 7px; border-radius: 999px; background: var(--ring); display: inline-block; } +.status-dot.healthy { background: var(--success); } +.status-dot.unhealthy { background: var(--destructive); } +.summary-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; } +.metric-card, .panel { border: 1px solid var(--border); border-radius: 14px; background: var(--card); } +.metric-card { padding: 18px; min-height: 116px; } +.metric-card .value { font-size: 29px; font-weight: 600; letter-spacing: -.035em; margin-top: 16px; } +.metric-card .caption { color: var(--muted-foreground); font-size: 12px; margin-top: 3px; } +.skeleton { background: linear-gradient(90deg, var(--card), var(--muted), var(--card)); background-size: 200% 100%; animation: shimmer 1.6s infinite; } +@keyframes shimmer { to { background-position: -200% 0; } } +.panel { padding: 18px; } +.topology-panel { margin-bottom: 36px; } +.panel-heading, .section-heading { display: flex; align-items: center; justify-content: space-between; gap: 16px; margin-bottom: 18px; } +.section-heading { margin-top: 34px; } +.topology { display: grid; grid-template-columns: 1fr 48px 1fr 48px 2fr; align-items: stretch; gap: 8px; } +.topology-node { border: 1px solid var(--border); background: var(--background); border-radius: 10px; padding: 14px; min-width: 0; } +.topology-node strong { display: block; font-size: 13px; } +.topology-node span { display: block; color: var(--muted-foreground); font-size: 12px; margin-top: 4px; overflow: hidden; text-overflow: ellipsis; } +.topology-cells { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; } +.connector { display: grid; place-items: center; color: var(--muted-foreground); } +.connector::before { content: ''; width: 100%; height: 1px; background: var(--border); } +.chart-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 36px; } +.chart-card { min-height: 220px; } +.chart-head { display: flex; justify-content: space-between; align-items: start; } +.chart-value { font-size: 22px; font-weight: 600; letter-spacing: -.03em; } +.chart-unit { color: var(--muted-foreground); font-size: 11px; } +.chart { width: 100%; height: 122px; margin-top: 18px; overflow: visible; } +.chart path.area { fill: color-mix(in srgb, var(--foreground) 5%, transparent); } +.chart path.line { fill: none; stroke: var(--foreground); stroke-width: 1.5; vector-effect: non-scaling-stroke; } +.chart line { stroke: var(--border); stroke-width: 1; } +.chart-empty { color: var(--muted-foreground); height: 120px; display: grid; place-items: center; font-size: 12px; } +.two-column { display: grid; grid-template-columns: 1.7fr 1fr; gap: 12px; margin-bottom: 12px; } +.three-column { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; } +.two-column > *, .three-column > *, .chart-grid > * { min-width: 0; } +.table-wrap { overflow-x: auto; } +table { width: 100%; border-collapse: collapse; font-size: 12px; } +th { color: var(--muted-foreground); font-weight: 500; text-align: left; padding: 0 10px 10px; } +td { padding: 12px 10px; border-top: 1px solid var(--border); vertical-align: middle; } +td:first-child, th:first-child { padding-left: 0; } +td:last-child, th:last-child { padding-right: 0; } +.mono { font-family: var(--font-mono); font-size: 11px; } +.badge { display: inline-flex; align-items: center; border: 1px solid var(--border); border-radius: 999px; padding: 2px 7px; font-size: 11px; color: var(--muted-foreground); } +.badge.healthy { color: var(--success); border-color: color-mix(in srgb, var(--success) 25%, var(--border)); } +.badge.unhealthy { color: var(--destructive); border-color: color-mix(in srgb, var(--destructive) 25%, var(--border)); } +.service-list, .list { display: grid; } +.service-row, .list-row { padding: 11px 0; border-top: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; gap: 12px; min-width: 0; } +.service-row:first-child, .list-row:first-child { border-top: 0; padding-top: 0; } +.service-row:last-child, .list-row:last-child { padding-bottom: 0; } +.row-title { font-size: 12px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.row-caption, .meta, .muted { color: var(--muted-foreground); font-size: 11px; } +.panel-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.45; margin: -8px 0 14px; } +.panel-link:hover { color: var(--foreground); text-decoration: underline; } +.row-caption { margin-top: 3px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +a { color: inherit; text-decoration: none; } +a:hover .row-title { text-decoration: underline; } +.cost-total { display: flex; align-items: baseline; gap: 7px; margin-bottom: 12px; } +.cost-total strong { font-size: 28px; letter-spacing: -.035em; } +.cost-line { display: flex; justify-content: space-between; gap: 10px; padding: 7px 0; border-top: 1px solid var(--border); font-size: 12px; } +.cost-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.5; margin-top: 12px; } +.banner { border: 1px solid var(--border); border-radius: 10px; padding: 11px 13px; margin-bottom: 12px; font-size: 12px; } +.banner.error { color: var(--destructive); } +.banner.warning { color: var(--warning); } +.hidden { display: none !important; } +.power-actions { display: flex; align-items: center; gap: 8px; margin-top: 14px; flex-wrap: wrap; } +footer { border-top: 1px solid var(--border); padding: 24px; color: var(--muted-foreground); font-size: 11px; text-align: center; } +@media (max-width: 1050px) { + .summary-grid, .chart-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .three-column { grid-template-columns: 1fr; } + .topology { grid-template-columns: 1fr; } + .connector { height: 20px; } + .connector::before { height: 100%; width: 1px; } +} +@media (max-width: 760px) { + .topbar { height: auto; min-height: 64px; padding: 12px 16px; align-items: stretch; flex-direction: column; gap: 12px; } + .brand div { display: grid; gap: 0; } + .toolbar { flex-wrap: wrap; justify-content: flex-start; } + main { padding: 28px 16px 56px; } + .page-heading { align-items: flex-start; flex-direction: column; } + .summary-grid, .chart-grid, .two-column { grid-template-columns: 1fr; } + .topology-cells { grid-template-columns: 1fr; } + table { min-width: 600px; } +} diff --git a/cloud/apps/relay-ops/src/cost-model.test.ts b/cloud/apps/relay-ops/src/cost-model.test.ts new file mode 100644 index 00000000000..9ee6f6adf9d --- /dev/null +++ b/cloud/apps/relay-ops/src/cost-model.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' +import { buildCostModel } from './cost-model.js' +import { + RELAY_OPS_ENVIRONMENTS, + relayOpsCellsFromTerraform, + type RelayOpsCellConfig +} from './environment-config.js' +import type { ResourceInventory } from './resource-inventory.js' + +const regionalCellsSource = ` +relay_gce_cells = { + "staging-gce-c3" = { + hostname = "c3" + zone = "us-central1-a" + machine_type = "e2-standard-2" + capacity_requests = 4000 + } + "staging-gce-c4" = { + hostname = "c4" + region = "asia-east2" + zone = "asia-east2-a" + machine_type = "e2-standard-4" + capacity_requests = 6000 + database_pool_max = 10 + initially_enabled = false + } +} +` + +function inventory( + targetSize: number, + activationPolicy: string, + cells: RelayOpsCellConfig[] = RELAY_OPS_ENVIRONMENTS.staging.cells +): ResourceInventory { + return { + director: null, + auth: null, + sql: { state: targetSize ? 'RUNNABLE' : 'STOPPED', activationPolicy, tier: 'db-custom-1-3840', availabilityType: 'ZONAL', databaseVersion: 'POSTGRES_17' }, + certificate: null, + directorEndpoint: { health: null, ready: null, latencyMs: null }, + authEndpoint: { health: null, ready: null, latencyMs: null }, + cells: cells.map((cell) => ({ + ...cell, migName: `mig-${cell.hostname}`, targetSize, runningInstances: targetSize, + stable: true, template: 'template', imageDigest: null, + backendHealth: targetSize ? 'healthy' : 'empty', + endpoint: { health: null, ready: null, latencyMs: null } + })), + warnings: [] + } +} + +describe('buildCostModel', () => { + it('shows the sleeping staging floor without VM or SQL compute', () => { + const result = buildCostModel(RELAY_OPS_ENVIRONMENTS.staging, inventory(0, 'NEVER')) + expect(result.kind).toBe('planning-estimate') + expect(result.monthlyUsd).toBe(34) + expect(result.lines.find((line) => line.label === 'Relay cell VMs')?.monthlyUsd).toBe(0) + expect(result.actualBilling.available).toBe(false) + expect(result.caveats[0]).toContain('not the Cloud Billing invoice') + }) + + it('prices durable machine inventory and network floors by region', () => { + const cells = relayOpsCellsFromTerraform({ + environment: 'staging', + domain: 'relay-staging.onorca.dev', + source: regionalCellsSource + }) + const environment = { ...RELAY_OPS_ENVIRONMENTS.staging, cells } + const result = buildCostModel(environment, inventory(1, 'NEVER', cells)) + const machines = result.lines.find((line) => line.label === 'Relay cell VMs') + const network = result.lines.find((line) => line.label === 'Load balancer and network floor') + + expect(machines).toEqual({ + label: 'Relay cell VMs', + monthlyUsd: 185.79, + basis: '2 configured VM cells at regional machine rates × 730 hours' + }) + + expect(network).toEqual({ + label: 'Load balancer and network floor', + monthlyUsd: 29, + basis: 'shared HTTPS foundation plus NAT floor in 2 configured regions' + }) + }) +}) diff --git a/cloud/apps/relay-ops/src/cost-model.ts b/cloud/apps/relay-ops/src/cost-model.ts new file mode 100644 index 00000000000..39e062597a8 --- /dev/null +++ b/cloud/apps/relay-ops/src/cost-model.ts @@ -0,0 +1,115 @@ +import type { + RelayOpsEnvironment, + RelayOpsMachineType, + RelayOpsRegion +} from './environment-config.js' +import type { ResourceInventory } from './resource-inventory.js' + +export type CostLine = { + label: string + monthlyUsd: number + basis: string +} + +export type CostModel = { + kind: 'planning-estimate' + monthlyUsd: number + rangeUsd: [number, number] + actualBilling: { available: false; reason: string } + lines: CostLine[] + caveats: string[] +} + +const HOURS_PER_MONTH = 730 +const MACHINE_HOURLY_USD: Record< + RelayOpsRegion, + Record +> = { + 'us-central1': { + 'e2-standard-2': 0.06701142, + 'e2-standard-4': 0.13402284 + }, + 'asia-east2': { + 'e2-standard-2': 0.0938, + 'e2-standard-4': 0.1875 + } +} +const SHARED_NETWORK_FLOOR_USD = 19 +const REGIONAL_NAT_FLOOR_USD = 5 + +function round(value: number): number { + return Math.round(value * 100) / 100 +} + +export function buildCostModel( + environment: RelayOpsEnvironment, + resources: ResourceInventory +): CostModel { + const runningCells = resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0) + const runningCellCount = runningCells.reduce((sum, cell) => sum + (cell.targetSize ?? 0), 0) + const compute = runningCells.reduce( + (sum, cell) => + sum + (cell.targetSize ?? 0) * MACHINE_HOURLY_USD[cell.region][cell.machineType], + 0 + ) * HOURS_PER_MONTH + const disks = runningCellCount * 30 * 0.1 + const sqlRunning = resources.sql?.activationPolicy === 'ALWAYS' + const sql = sqlRunning ? (environment.id === 'production' ? 105 : 52) : 0 + const cloudRunMinimums = + (resources.director?.minInstances ?? 0) + (resources.auth?.minInstances ?? 0) + const cloudRun = cloudRunMinimums * 10 + const configuredRegions = new Set( + (resources.cells.length > 0 ? resources.cells : environment.cells).map((cell) => cell.region) + ) + const networkFoundation = + SHARED_NETWORK_FLOOR_USD + configuredRegions.size * REGIONAL_NAT_FLOOR_USD + const observability = environment.id === 'production' ? 12 : 5 + const lines: CostLine[] = [ + { + label: 'Relay cell VMs', + monthlyUsd: round(compute), + basis: `${runningCellCount} configured VM cells at regional machine rates × 730 hours` + }, + { + label: 'Cell boot disks', + monthlyUsd: round(disks), + basis: `${runningCellCount} × 30 GB balanced persistent disk` + }, + { + label: 'Cloud SQL', + monthlyUsd: sql, + basis: sqlRunning ? `${resources.sql?.tier ?? 'configured tier'} active` : 'stopped' + }, + { + label: 'Cloud Run minimums', + monthlyUsd: cloudRun, + basis: `${cloudRunMinimums} configured minimum instances` + }, + { + label: 'Load balancer and network floor', + monthlyUsd: networkFoundation, + basis: `shared HTTPS foundation plus NAT floor in ${configuredRegions.size} configured region${configuredRegions.size === 1 ? '' : 's'}` + }, + { + label: 'Logs and monitoring allowance', + monthlyUsd: observability, + basis: 'planning allowance; varies with traffic and retention' + } + ] + const monthlyUsd = round(lines.reduce((sum, line) => sum + line.monthlyUsd, 0)) + return { + kind: 'planning-estimate', + monthlyUsd, + rangeUsd: [round(monthlyUsd * 0.85), round(monthlyUsd * 1.3)], + actualBilling: { + available: false, + reason: 'Cloud Billing export is not configured in either Relay project.' + }, + lines, + caveats: [ + 'This is a modeled run-rate, not the Cloud Billing invoice.', + 'Network egress, actual request traffic, credits, discounts, taxes, and free tiers are excluded.', + 'Use the GCP Billing report for authoritative spend and forecasts.' + ] + } +} diff --git a/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts b/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts new file mode 100644 index 00000000000..247b4b53ed0 --- /dev/null +++ b/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest' +import { DashboardSnapshotCache } from './dashboard-snapshot.js' +import type { DashboardSnapshot } from './dashboard-snapshot.js' +import type { GcloudClient } from './gcloud-client.js' + +const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) } + +function snapshot(kind: 'good' | 'unavailable'): DashboardSnapshot { + const good = kind === 'good' + return { + generatedAt: '2026-07-15T12:00:00.000Z', + resources: { + director: good ? {} : null, + auth: good ? {} : null, + sql: good ? {} : null, + cells: [{ targetSize: good ? 1 : null }] + }, + monitoring: { + warnings: good + ? [] + : ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.'] + }, + summary: { observedConnections: good ? 7 : 0 }, + warnings: good ? [] : ['Google Cloud credentials are unavailable. Run gcloud auth login.'], + stale: false, + staleReason: null + } as unknown as DashboardSnapshot +} + +describe('DashboardSnapshotCache', () => { + it('keeps the last good view when a later credential refresh fails', async () => { + let calls = 0 + const cache = new DashboardSnapshotCache(gcloud, 0, async () => { + calls += 1 + return snapshot(calls === 1 ? 'good' : 'unavailable') + }) + + const first = await cache.read('production', 30) + const second = await cache.read('production', 31) + + expect(first.stale).toBe(false) + expect(second.stale).toBe(true) + expect(second.summary.observedConnections).toBe(7) + expect(second.staleReason).toContain('credentials') + }) + + it('keeps the last good view when a later collector throws', async () => { + let calls = 0 + const cache = new DashboardSnapshotCache(gcloud, 0, async () => { + calls += 1 + if (calls > 1) throw new Error('sensitive collector context') + return snapshot('good') + }) + + await cache.read('production', 30) + const second = await cache.read('production', 31) + + expect(second.stale).toBe(true) + expect(second.summary.observedConnections).toBe(7) + expect(JSON.stringify(second)).not.toContain('sensitive collector context') + }) +}) diff --git a/cloud/apps/relay-ops/src/dashboard-snapshot.ts b/cloud/apps/relay-ops/src/dashboard-snapshot.ts new file mode 100644 index 00000000000..0d6f0c1151b --- /dev/null +++ b/cloud/apps/relay-ops/src/dashboard-snapshot.ts @@ -0,0 +1,167 @@ +import type { RelayOpsEnvironmentId } from './environment-config.js' +import { relayOpsEnvironment } from './environment-config.js' +import type { GcloudClient } from './gcloud-client.js' +import { readRelayWorkflowRuns } from './github-runs.js' +import { readMonitoringSnapshot } from './monitoring-snapshot.js' +import { readResourceInventory } from './resource-inventory.js' +import { buildCostModel } from './cost-model.js' + +export type DashboardSnapshot = Awaited> + +export async function buildDashboardSnapshot( + environmentId: RelayOpsEnvironmentId, + gcloud: GcloudClient, + options: { windowMinutes?: number; fetchImpl?: typeof fetch; now?: Date } = {} +) { + const generatedAt = (options.now ?? new Date()).toISOString() + const environment = relayOpsEnvironment(environmentId) + const [monitoringResult, resourceResult, workflowResult] = await Promise.allSettled([ + readMonitoringSnapshot(environment, gcloud, { + ...(options.now === undefined ? {} : { now: options.now }), + ...(options.windowMinutes === undefined ? {} : { windowMinutes: options.windowMinutes }), + ...(options.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl }) + }), + readResourceInventory(environment, gcloud, options.fetchImpl), + readRelayWorkflowRuns() + ]) + if (monitoringResult.status === 'rejected' || resourceResult.status === 'rejected') { + const failed = [ + monitoringResult.status === 'rejected' ? 'Monitoring snapshot' : null, + resourceResult.status === 'rejected' ? 'Resource inventory' : null + ].filter(Boolean) + throw new Error(`${failed.join(' and ')} unavailable`) + } + const resources = resourceResult.value + const monitoring = monitoringResult.value + const warnings = [...resources.warnings, ...monitoring.warnings] + const poweredDigests = new Set( + resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0).map((cell) => cell.imageDigest) + ) + if (poweredDigests.has(null)) warnings.push('A powered cell image digest is unavailable.') + if (poweredDigests.size > 1) warnings.push('Powered cells are not serving one immutable digest.') + const expectedCertificateDomain = `*.${new URL(environment.cells[0]!.origin).hostname + .split('.').slice(1).join('.')}` + if (resources.certificate && !resources.certificate.domains.includes(expectedCertificateDomain)) { + warnings.push('The Relay certificate domain does not match the configured cell domain.') + } + if (workflowResult.status === 'rejected') warnings.push('GitHub workflow history is unavailable.') + const observedConnections = monitoring.metrics.total_connections.latest ?? 0 + const observedControls = monitoring.metrics.controls.latest ?? 0 + const observedSplices = monitoring.metrics.splices.latest ?? 0 + const configuredCapacity = environment.cells + .filter((cell) => cell.configuredAdmission) + .reduce((sum, cell) => sum + cell.capacityRequests, 0) + const cellInventoryAvailable = resources.cells.every((cell) => cell.targetSize !== null) + const poweredCapacity = cellInventoryAvailable + ? resources.cells + .filter((cell) => (cell.targetSize ?? 0) > 0) + .reduce((sum, cell) => sum + cell.capacityRequests, 0) + : null + return { + schemaVersion: 1, + generatedAt, + environment: { + id: environment.id, + label: environment.label, + project: environment.project, + region: environment.region, + directorOrigin: environment.directorOrigin, + authOrigin: environment.authOrigin, + consoleLinks: { + project: `https://console.cloud.google.com/home/dashboard?project=${environment.project}`, + alerts: `https://console.cloud.google.com/monitoring/alerting?project=${environment.project}`, + compute: `https://console.cloud.google.com/compute/instanceGroups/list?project=${environment.project}` + } + }, + summary: { + observedConnections, + observedControls, + observedSplices, + configuredCapacity, + poweredCapacity, + activeCells: cellInventoryAvailable + ? resources.cells.filter( + (cell) => (cell.targetSize ?? 0) > 0 && cell.backendHealth === 'healthy' + ).length + : null, + totalCells: resources.cells.length + }, + resources, + monitoring, + workflows: workflowResult.status === 'fulfilled' ? workflowResult.value : [], + cost: buildCostModel(environment, resources), + warnings, + stale: false, + staleReason: null as string | null + } +} + +type SnapshotCacheEntry = { snapshot: DashboardSnapshot; expiresAt: number } +type SnapshotBuilder = ( + environment: RelayOpsEnvironmentId, + gcloud: GcloudClient, + options: { windowMinutes?: number } +) => Promise + +export class DashboardSnapshotCache { + private readonly entries = new Map() + private readonly pending = new Map>() + private readonly lastGood = new Map() + + constructor( + private readonly gcloud: GcloudClient, + private readonly ttlMs = 30_000, + private readonly builder: SnapshotBuilder = buildDashboardSnapshot + ) {} + + async read(environment: RelayOpsEnvironmentId, windowMinutes: number): Promise { + const key = `${environment}:${windowMinutes}` + const cached = this.entries.get(key) + if (cached && cached.expiresAt > Date.now()) return cached.snapshot + const existing = this.pending.get(key) + if (existing) return await existing + const request = this.builder(environment, this.gcloud, { windowMinutes }) + .then((snapshot) => { + const coreInventoryUnavailable = + snapshot.resources.director === null && + snapshot.resources.auth === null && + snapshot.resources.sql === null && + snapshot.resources.cells.every((cell) => cell.targetSize === null) + const monitoringCredentialsUnavailable = snapshot.monitoring.warnings.some( + (warning) => warning.includes('credentials are unavailable') + ) + const lastGood = this.lastGood.get(environment) + const result = (coreInventoryUnavailable || monitoringCredentialsUnavailable) && lastGood + ? { + ...lastGood, + stale: true, + staleReason: 'Local Google Cloud credentials are temporarily unavailable.', + warnings: [...new Set([...lastGood.warnings, ...snapshot.warnings])] + } + : snapshot + if (!coreInventoryUnavailable && !monitoringCredentialsUnavailable) { + this.lastGood.set(environment, snapshot) + } + this.entries.set(key, { snapshot: result, expiresAt: Date.now() + this.ttlMs }) + return result + }) + .catch((error: unknown) => { + const lastGood = this.lastGood.get(environment) + if (!lastGood) throw error + const stale = { + ...lastGood, + stale: true, + staleReason: 'The latest operations refresh failed; showing the last good snapshot.', + warnings: [...new Set([ + ...lastGood.warnings, + 'The latest operations refresh failed before a complete snapshot was available.' + ])] + } + this.entries.set(key, { snapshot: stale, expiresAt: Date.now() + this.ttlMs }) + return stale + }) + .finally(() => this.pending.delete(key)) + this.pending.set(key, request) + return await request + } +} diff --git a/cloud/apps/relay-ops/src/environment-config.test.ts b/cloud/apps/relay-ops/src/environment-config.test.ts new file mode 100644 index 00000000000..321167d7cc8 --- /dev/null +++ b/cloud/apps/relay-ops/src/environment-config.test.ts @@ -0,0 +1,148 @@ +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { + RELAY_OPS_ENVIRONMENTS, + relayOpsCellsFromTerraform +} from './environment-config.js' + +const durableUsCell = ` + "production-gce-c26" = { + hostname = "c26" + zone = "us-central1-a" + machine_type = "e2-standard-4" + capacity_requests = 4000 + initially_enabled = false + } +` + +const durableAsiaCells = ` + "production-gce-c27" = { + hostname = "c27" + region = "asia-east2" + zone = "asia-east2-a" + machine_type = "e2-standard-4" + capacity_requests = 6000 + database_pool_max = 10 + initially_enabled = false + } + "production-gce-c28" = { + hostname = "c28" + region = "asia-east2" + zone = "asia-east2-b" + machine_type = "e2-standard-4" + capacity_requests = 6000 + database_pool_max = 10 + initially_enabled = false + } + "production-gce-c29" = { + hostname = "c29" + region = "asia-east2" + zone = "asia-east2-c" + machine_type = "e2-standard-4" + capacity_requests = 6000 + database_pool_max = 10 + initially_enabled = false + } +` + +const durableAsiaSource = ` +relay_gce_cells = {${durableUsCell}${durableAsiaCells}} +` + +const durableUsOnlySource = ` +relay_gce_cells = {${durableUsCell}} +` + +// Why: relay-ops sat at 18 cells for four days after C19-C22 shipped, which threw +// `selector membership must contain every configured cell exactly once` and blocked +// every production mutation. Reading Terraform here makes that drift fail the build. +function terraformCells(environment: 'production' | 'staging'): Array<{ + cellId: string + region: string + zone: string + machineType: string + capacityRequests: number + databasePoolMax: number +}> { + const tfvars = readFileSync( + fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)), + 'utf8' + ) + const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(tfvars)?.[1] ?? '' + return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)] + .map((match) => ({ + cellId: match[1] ?? '', + region: /region\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? 'us-central1', + zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', + machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', + capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]), + databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10) + })) + .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId)) +} + +const cellOrdinal = (cellId: string): number => Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0) + +describe('relay operations environment config', () => { + it.each(['production', 'staging'] as const)( + 'matches the %s cells Terraform actually provisions', + (environment) => { + const expected = terraformCells(environment) + expect(expected.length).toBeGreaterThan(0) + expect( + RELAY_OPS_ENVIRONMENTS[environment].cells.map((cell) => ({ + cellId: cell.cellId, + region: cell.region, + zone: cell.zone, + machineType: cell.machineType, + capacityRequests: cell.capacityRequests, + databasePoolMax: cell.databasePoolMax + })) + ).toEqual(expected) + } + ) + + it('derives hostname and origin from the cell ordinal', () => { + const cells = RELAY_OPS_ENVIRONMENTS.production.cells + expect(cells.at(-1)).toMatchObject({ + hostname: `c${cells.length}`, + origin: `https://c${cells.length}.relay.onorca.dev` + }) + }) + + it('inventories Asia cells only when they exist in durable Terraform', () => { + const cells = relayOpsCellsFromTerraform({ + environment: 'production', + domain: 'relay.onorca.dev', + source: durableAsiaSource + }) + + expect(cells.slice(1)).toEqual([ + { + cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev', + region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4', + capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + }, + { + cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev', + region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4', + capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + }, + { + cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev', + region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4', + capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + } + ]) + + const usOnlyCells = relayOpsCellsFromTerraform({ + environment: 'production', + domain: 'relay.onorca.dev', + source: durableUsOnlySource + }) + expect(usOnlyCells).toHaveLength(1) + expect(usOnlyCells.some((cell) => cell.region === 'asia-east2')).toBe(false) + expect(usOnlyCells.some((cell) => cell.cellId === 'production-gce-c27')).toBe(false) + }) +}) diff --git a/cloud/apps/relay-ops/src/environment-config.ts b/cloud/apps/relay-ops/src/environment-config.ts new file mode 100644 index 00000000000..da3de7765ed --- /dev/null +++ b/cloud/apps/relay-ops/src/environment-config.ts @@ -0,0 +1,125 @@ +import { readFileSync } from 'node:fs' +import { z } from 'zod' + +export type RelayOpsEnvironmentId = 'production' | 'staging' +export type RelayOpsRegion = 'us-central1' | 'asia-east2' +export type RelayOpsMachineType = 'e2-standard-2' | 'e2-standard-4' + +export type RelayOpsCellConfig = { + cellId: string + hostname: string + origin: string + region: RelayOpsRegion + zone: string + machineType: RelayOpsMachineType + capacityRequests: number + databasePoolMax: number + configuredAdmission: boolean +} + +export type RelayOpsEnvironment = { + id: RelayOpsEnvironmentId + label: string + project: string + region: string + directorOrigin: string + authOrigin: string + directorService: string + authService: string + sqlInstance: string + migPrefix: string + certificateName: string + cells: RelayOpsCellConfig[] +} + +const RegionSchema = z.enum(['us-central1', 'asia-east2']) +const MachineTypeSchema = z.enum(['e2-standard-2', 'e2-standard-4']) +const EnvironmentSchema = z.enum(['production', 'staging']) + +function cellOrdinal(cellId: string): number { + return Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0) +} + +function required(body: string, pattern: RegExp, label: string): string { + const value = pattern.exec(body)?.[1] + if (!value) throw new Error(`Relay Ops could not read ${label} from durable Terraform config`) + return value +} + +export function relayOpsCellsFromTerraform(input: { + environment: RelayOpsEnvironmentId + domain: string + source: string +}): RelayOpsCellConfig[] { + const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(input.source)?.[1] + if (!block) throw new Error('Relay Ops could not read durable Relay cells') + return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)] + .map((match) => { + const cellId = match[1]! + const body = match[2]! + const hostname = required(body, /\bhostname\s*=\s*"([^"]+)"/, `${cellId} hostname`) + const configuredAdmission = /\binitially_enabled\s*=\s*(true|false)/.exec(body)?.[1] + return { + cellId, + hostname, + origin: `https://${hostname}.${input.domain}`, + region: RegionSchema.parse( + /\bregion\s*=\s*"([^"]+)"/.exec(body)?.[1] ?? 'us-central1' + ), + zone: required(body, /\bzone\s*=\s*"([^"]+)"/, `${cellId} zone`), + machineType: MachineTypeSchema.parse( + required(body, /\bmachine_type\s*=\s*"([^"]+)"/, `${cellId} machine type`) + ), + capacityRequests: Number( + required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`) + ), + databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10), + configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true' + } + }) + .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId)) +} + +function durableCells(environment: RelayOpsEnvironmentId, domain: string): RelayOpsCellConfig[] { + const source = readFileSync( + // Repository root; infra/terraform moves with this tree, so the relative depth holds. + new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url), + 'utf8' + ) + return relayOpsCellsFromTerraform({ environment, domain, source }) +} + +export const RELAY_OPS_ENVIRONMENTS: Record = { + production: { + id: 'production', + label: 'Production', + project: 'onorca-cloud', + region: 'us-central1', + directorOrigin: 'https://relay.onorca.dev', + authOrigin: 'https://login.onorca.dev', + directorService: 'orca-cloud-relay', + authService: 'orca-cloud-auth', + sqlInstance: 'orca-cloud-auth-db', + migPrefix: 'orca-cloud-relay-gce-', + certificateName: 'orca-cloud-relay-gce', + cells: durableCells('production', 'relay.onorca.dev') + }, + staging: { + id: 'staging', + label: 'Staging', + project: 'onorca-cloud-staging', + region: 'us-central1', + directorOrigin: 'https://relay-staging.onorca.dev', + authOrigin: 'https://auth-staging.onorca.dev', + directorService: 'orca-cloud-relay-staging', + authService: 'orca-cloud-auth-staging', + sqlInstance: 'orca-cloud-staging-auth-db', + migPrefix: 'orca-cloud-staging-relay-gce-', + certificateName: 'orca-cloud-staging-relay-gce', + cells: durableCells('staging', 'relay-staging.onorca.dev') + } +} + +export function relayOpsEnvironment(value: unknown): RelayOpsEnvironment { + return RELAY_OPS_ENVIRONMENTS[EnvironmentSchema.parse(value)] +} diff --git a/cloud/apps/relay-ops/src/gcloud-client.test.ts b/cloud/apps/relay-ops/src/gcloud-client.test.ts new file mode 100644 index 00000000000..3347c0cb367 --- /dev/null +++ b/cloud/apps/relay-ops/src/gcloud-client.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { createGcloudClient } from './gcloud-client.js' + +describe('createGcloudClient', () => { + it('shares and caches one credential refresh across concurrent readers', async () => { + let calls = 0 + const token = 'a'.repeat(40) + const client = createGcloudClient(async () => { + calls += 1 + await Promise.resolve() + return token + }) + + const values = await Promise.all([ + client.accessToken(), + client.accessToken(), + client.accessToken() + ]) + + expect(values).toEqual([token, token, token]) + expect(await client.accessToken()).toBe(token) + expect(calls).toBe(1) + }) + + it('caches bounded identity tokens by audience', async () => { + const commands: string[][] = [] + const token = 'aaa.bbb.ccc' + const client = createGcloudClient(async (args) => { + commands.push(args) + return token + }) + await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token) + await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token) + expect(commands).toEqual([ + [ + 'auth', + 'print-identity-token', + '--audiences=https://relay.example/admin', + '--include-email' + ] + ]) + }) +}) diff --git a/cloud/apps/relay-ops/src/gcloud-client.ts b/cloud/apps/relay-ops/src/gcloud-client.ts new file mode 100644 index 00000000000..b565242ec0e --- /dev/null +++ b/cloud/apps/relay-ops/src/gcloud-client.ts @@ -0,0 +1,77 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const TOKEN_PATTERN = /^[A-Za-z0-9._~+\/-]{32,8192}$/ + +export type GcloudClient = { + accessToken(): Promise + identityToken?(audience: string): Promise +} + +export class GcloudCommandError extends Error { + constructor(readonly operation: string) { + super(`${operation} is unavailable`) + } +} + +async function runGcloud(args: string[]): Promise { + try { + const result = await execFileAsync('gcloud', args, { + encoding: 'utf8', + timeout: 90_000, + maxBuffer: 8 * 1024 * 1024, + env: { + ...process.env, + CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK: '1', + CLOUDSDK_CORE_DISABLE_PROMPTS: '1', + CLOUDSDK_CORE_DISABLE_USAGE_REPORTING: '1' + } + }) + return result.stdout.trim() + } catch { + // Gcloud stderr can echo command context; keep dashboard errors intentionally non-sensitive. + throw new GcloudCommandError(`gcloud ${args.slice(0, 3).join(' ')}`) + } +} + +export function createGcloudClient( + tokenCommand: (args: string[]) => Promise = runGcloud +): GcloudClient { + let cachedToken: { value: string; expiresAt: number } | null = null + const identityTokens = new Map() + let pendingToken: Promise | null = null + return { + async accessToken(): Promise { + if (cachedToken && cachedToken.expiresAt > Date.now()) return cachedToken.value + if (pendingToken) return await pendingToken + // One refresh avoids concurrent gcloud processes contending on the local credential store. + pendingToken = tokenCommand(['auth', 'print-access-token']) + .then((token) => { + if (!TOKEN_PATTERN.test(token)) throw new GcloudCommandError('gcloud access token') + cachedToken = { value: token, expiresAt: Date.now() + 5 * 60_000 } + return token + }) + .finally(() => { pendingToken = null }) + return await pendingToken + }, + async identityToken(audience: string): Promise { + const cached = identityTokens.get(audience) + if (cached && cached.expiresAt > Date.now()) return cached.value + const token = await tokenCommand([ + 'auth', + 'print-identity-token', + `--audiences=${audience}`, + '--include-email' + ]) + if ( + token.length > 8_192 || + !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token) + ) { + throw new GcloudCommandError('gcloud identity token') + } + identityTokens.set(audience, { value: token, expiresAt: Date.now() + 5 * 60_000 }) + return token + } + } +} diff --git a/cloud/apps/relay-ops/src/github-runs.ts b/cloud/apps/relay-ops/src/github-runs.ts new file mode 100644 index 00000000000..47ef8469fa9 --- /dev/null +++ b/cloud/apps/relay-ops/src/github-runs.ts @@ -0,0 +1,77 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { z } from 'zod' +import { relayRepositoryApiPath } from './relay-repository.js' + +const execFileAsync = promisify(execFile) + +const WorkflowRunSchema = z.object({ + id: z.number().int().positive(), + name: z.string(), + event: z.string(), + status: z.string(), + conclusion: z.string().nullable(), + head_sha: z.string(), + html_url: z.string().url(), + created_at: z.string(), + updated_at: z.string() +}) + +const WorkflowRunsSchema = z.object({ + workflow_runs: z.array(WorkflowRunSchema) +}) + +export type RelayWorkflowRun = { + id: number + name: string + status: string + conclusion: string | null + headSha: string + url: string + createdAt: string + updatedAt: string +} + +const RELAY_WORKFLOW_PATTERN = /(Relay|Auth|Power)/i + +export async function readRelayWorkflowRuns(): Promise { + let stdout: string + try { + const result = await execFileAsync( + 'gh', + [ + 'api', + '--method', + 'GET', + relayRepositoryApiPath('actions/runs'), + '-f', + 'per_page=100' + ], + { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 } + ) + stdout = result.stdout + } catch { + throw new Error('GitHub workflow history is unavailable') + } + const runs = WorkflowRunsSchema.parse(JSON.parse(stdout) as unknown).workflow_runs + const counts = new Map() + return runs + .filter((run) => { + if (!RELAY_WORKFLOW_PATTERN.test(run.name)) return false + const count = counts.get(run.name) ?? 0 + if (count >= 2) return false + counts.set(run.name, count + 1) + return true + }) + .slice(0, 12) + .map((run) => ({ + id: run.id, + name: run.name, + status: run.status, + conclusion: run.conclusion, + headSha: run.head_sha.slice(0, 8), + url: run.html_url, + createdAt: run.created_at, + updatedAt: run.updated_at + })) +} diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts new file mode 100644 index 00000000000..18ee4495078 --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts @@ -0,0 +1,373 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + livePreflightGcloud, + runIncidentLivePreflight +} from './incident-live-preflight-cli.js' +import type { IncidentSample } from './incident-monitor.js' +import type { AdmissionSelector } from './incident-selector.js' + +const directories: string[] = [] +const now = Date.parse('2026-07-28T12:00:00.000Z') +const selector = { + generation: 1, + membership: { + existingOnly: ['production-gce-c1'], + migrationOnly: [], + general: [] + } +} + +function stateFile( + migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition' = 'strict', + overrides: Record = {} +): string { + const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-')) + directories.push(directory) + const path = join(directory, 'state.json') + const expectedSelector = migrationPolicy === 'capacity-transition' + ? { + generation: 1, + membership: { + existingOnly: [], + migrationOnly: [], + general: ['production-gce-c1'] + } + } + : selector + writeFileSync(path, JSON.stringify({ + schemaVersion: 4, + environment: 'production', + expectedSelector, + migrationPolicy, + recoverySourceCellId: + migrationPolicy === 'recover-forward' ? 'production-gce-c1' : null, + capacityCellId: + migrationPolicy === 'capacity-transition' ? 'production-gce-c1' : null, + preDrainDryRun: true, + startedAt: new Date(now - 17 * 60_000).toISOString(), + windowStartedAt: new Date(now - 16 * 60_000).toISOString(), + durationMinutes: 15, + intervalMs: 60_000, + sampleCount: 16, + lastSampleAt: new Date(now - 60_007).toISOString(), + frozenAt: null, + completedAt: new Date(now - 60_000).toISOString(), + ...overrides + })) + return path +} + +function sample(): IncidentSample { + const observedAt = new Date(now).toISOString() + const signal = (value: number) => ({ value, observedAt }) + return { + collectedAt: observedAt, + selector, + expectedSelector: selector, + cells: [{ + cellId: 'production-gce-c1', + runtimeKnown: true, + powered: true, + expectedAdmissionState: 'existing-only' + }], + sources: { + 'active-probe': { + observedAt, + signals: { + 'director.health': signal(1), + 'director.ready': signal(1), + 'director.latency_ms': signal(1), + 'auth.health': signal(1), + 'auth.ready': signal(1), + 'auth.latency_ms': signal(1), + 'cell.production-gce-c1.health': signal(1), + 'cell.production-gce-c1.ready': signal(1), + 'cell.production-gce-c1.latency_ms': signal(1) + } + }, + 'cloud-monitoring': { + observedAt, + signals: { + 'cloud_sql.cpu': signal(0.1), + 'cloud_sql.memory': signal(0.1), + 'cloud_sql.backends': signal(1), + 'cloud_sql.lock_waits': signal(0), + 'cloud_sql.deadlocks': signal(0), + 'director.instances': signal(5), + 'director.cpu': signal(0.1), + 'director.memory': signal(0.1), + 'director.concurrency': signal(1), + 'director.errors': signal(0), + 'auth.errors': signal(0) + } + }, + 'relay-logs': { + observedAt, + signals: { + 'relay.pool_waiting': signal(0), + 'relay.pool_wait_ms': signal(0), + 'relay.postgres_retries': signal(0), + 'relay.postgres_retry_exhausted': signal(0), + 'cell.production-gce-c1.connections': signal(1), + 'cell.production-gce-c1.queued_bytes': signal(0) + } + }, + 'director-admin': { + observedAt, + signals: { + 'cell.production-gce-c1.admission_state': signal(0), + 'cell.production-gce-c1.heartbeat_fresh': signal(1), + 'cell.production-gce-c1.heartbeat_age_ms': signal(1), + 'cell.production-gce-c1.migration_blocked': signal(0), + 'cell.production-gce-c1.migration_target_inactive': signal(0) + } + } + } + } +} + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('relay incident live preflight', () => { + it('accepts the package-manager argument separator', async () => { + await expect(runIncidentLivePreflight( + ['--', '--state-file', stateFile()], + { now: () => now, collect: async () => sample() } + )).resolves.toBeUndefined() + }) + + it('accepts one complete fresh green sample', async () => { + await expect(runIncidentLivePreflight( + ['--state-file', stateFile()], + { now: () => now, collect: async () => sample() } + )).resolves.toBeUndefined() + }) + + it('rejects monitor evidence beyond the 25-minute lineage bound', async () => { + const path = stateFile('strict', { + startedAt: new Date(now - 26 * 60_000 - 1).toISOString() + }) + await expect(runIncidentLivePreflight( + ['--state-file', path], + { now: () => now, collect: async () => sample() } + )).rejects.toThrow('monitor evidence is incomplete or stale') + }) + + it('scales the evidence age bound by same-cap wave index', async () => { + const agedState = (ageMs: number) => stateFile('strict', { + startedAt: new Date(now - ageMs - 17 * 60_000).toISOString(), + windowStartedAt: new Date(now - ageMs - 16 * 60_000).toISOString(), + lastSampleAt: new Date(now - ageMs - 7).toISOString(), + completedAt: new Date(now - ageMs).toISOString() + }) + const deps = { now: () => now, collect: async () => sample() } + // One predecessor cell roll (~16 min) exceeds wave 0 but fits wave 1. + const oneRollOld = agedState(17 * 60_000) + await expect(runIncidentLivePreflight( + ['--state-file', oneRollOld], deps + )).rejects.toThrow('monitor evidence is incomplete or stale') + await expect(runIncidentLivePreflight( + ['--state-file', oneRollOld, '--wave-index', '0'], deps + )).rejects.toThrow('monitor evidence is incomplete or stale') + await expect(runIncidentLivePreflight( + ['--state-file', oneRollOld, '--wave-index', '1'], deps + )).resolves.toBeUndefined() + // Both edges of one predecessor job timeout: 5min + 75min exactly. + await expect(runIncidentLivePreflight( + ['--state-file', agedState(80 * 60_000), '--wave-index', '1'], deps + )).resolves.toBeUndefined() + await expect(runIncidentLivePreflight( + ['--state-file', agedState(80 * 60_000 + 1), '--wave-index', '1'], deps + )).rejects.toThrow('monitor evidence is incomplete or stale') + await expect(runIncidentLivePreflight( + ['--state-file', agedState(155 * 60_000), '--wave-index', '2'], deps + )).resolves.toBeUndefined() + await expect(runIncidentLivePreflight( + ['--state-file', agedState(155 * 60_000 + 1), '--wave-index', '2'], deps + )).rejects.toThrow('monitor evidence is incomplete or stale') + await expect(runIncidentLivePreflight( + ['--state-file', agedState(230 * 60_000), '--wave-index', '3'], deps + )).resolves.toBeUndefined() + await expect(runIncidentLivePreflight( + ['--state-file', agedState(230 * 60_000 + 1), '--wave-index', '3'], deps + )).rejects.toThrow('monitor evidence is incomplete or stale') + // The wave index is a strict single-use 0-3 argument. + await expect(runIncidentLivePreflight( + ['--state-file', stateFile(), '--wave-index', '4'], deps + )).rejects.toThrow('usage:') + await expect(runIncidentLivePreflight( + ['--state-file', stateFile(), '--wave-index', ''], deps + )).rejects.toThrow('usage:') + await expect(runIncidentLivePreflight( + ['--state-file', stateFile(), '--wave-index', '1', '--wave-index', '1'], + deps + )).rejects.toThrow('usage:') + }) + + it('expects the wave-adjusted live selector generation', async () => { + const agedPath = stateFile('strict', { + startedAt: new Date(now - 34 * 60_000).toISOString(), + windowStartedAt: new Date(now - 33 * 60_000).toISOString(), + lastSampleAt: new Date(now - 17 * 60_000 - 7).toISOString(), + completedAt: new Date(now - 17 * 60_000).toISOString() + }) + const liveAt = (generation: number) => + async (expectedSelector: AdmissionSelector) => ({ + ...sample(), + selector: { ...selector, generation }, + expectedSelector + }) + // One predecessor roll advanced the live selector by exactly 2. + await expect(runIncidentLivePreflight( + ['--state-file', agedPath, '--wave-index', '1'], + { now: () => now, collect: liveAt(selector.generation + 2) } + )).resolves.toBeUndefined() + // The sealed pre-roll generation must no longer satisfy wave 1. + await expect(runIncidentLivePreflight( + ['--state-file', agedPath, '--wave-index', '1'], + { now: () => now, collect: liveAt(selector.generation) } + )).rejects.toThrow('director-admin/selector_mismatch') + // Wave 0 still expects the sealed generation itself. + await expect(runIncidentLivePreflight( + ['--state-file', stateFile()], + { now: () => now, collect: liveAt(selector.generation) } + )).resolves.toBeUndefined() + }) + + it('fails closed on a live threshold breach', async () => { + const unhealthy = sample() + unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9 + await expect(runIncidentLivePreflight( + ['--state-file', stateFile()], + { now: () => now, collect: async () => unhealthy } + )).rejects.toThrow('cloud-monitoring/threshold_max') + }) + + it('enforces the signed migration policy', async () => { + const inactiveTarget = sample() + inactiveTarget.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_target_inactive' + ]!.value = 30 + await expect(runIncidentLivePreflight( + ['--state-file', stateFile()], + { now: () => now, collect: async () => inactiveTarget } + )).rejects.toThrow('director-admin/threshold_max') + await expect(runIncidentLivePreflight( + ['--state-file', stateFile('recover-forward')], + { now: () => now, collect: async () => inactiveTarget } + )).resolves.toBeUndefined() + inactiveTarget.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_blocked' + ]!.value = 1 + await expect(runIncidentLivePreflight( + ['--state-file', stateFile('recover-forward')], + { now: () => now, collect: async () => inactiveTarget } + )).rejects.toThrow('director-admin/threshold_max') + }) + + it('binds capacity-transition evidence to its general cell', async () => { + const capacitySample = sample() + const capacitySelector = { + generation: 1, + membership: { + existingOnly: [], + migrationOnly: [], + general: ['production-gce-c1'] + } + } + capacitySample.selector = capacitySelector + capacitySample.expectedSelector = capacitySelector + capacitySample.cells[0]!.expectedAdmissionState = 'general' + capacitySample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.admission_state' + ]!.value = 2 + await expect(runIncidentLivePreflight( + ['--state-file', stateFile('capacity-transition')], + { now: () => now, collect: async () => capacitySample } + )).resolves.toBeUndefined() + capacitySample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_target_inactive' + ]!.value = 1 + await expect(runIncidentLivePreflight( + ['--state-file', stateFile('capacity-transition')], + { now: () => now, collect: async () => capacitySample } + )).rejects.toThrow('director-admin/threshold_max') + }) + + it('rejects stale live evidence', async () => { + const stale = sample() + stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString() + await expect(runIncidentLivePreflight( + ['--state-file', stateFile()], + { now: () => now, collect: async () => stale } + )).rejects.toThrow('active-probe/source_stale') + }) + + it('retries freshness-only failures when explicitly requested', async () => { + const stale = sample() + stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = + new Date(now - 180_001).toISOString() + const missing = sample() + delete missing.sources['relay-logs'] + const collect = vi.fn() + .mockResolvedValueOnce(stale) + .mockResolvedValueOnce(missing) + .mockResolvedValueOnce(sample()) + const wait = vi.fn(async () => undefined) + await expect(runIncidentLivePreflight( + ['--state-file', stateFile(), '--retry-freshness'], + { now: () => now, collect, wait } + )).resolves.toBeUndefined() + expect(collect).toHaveBeenCalledTimes(3) + expect(wait).toHaveBeenCalledTimes(2) + expect(wait).toHaveBeenNthCalledWith(1, 15_000) + expect(wait).toHaveBeenNthCalledWith(2, 15_000) + }) + + it('does not retry a threshold failure', async () => { + const unhealthy = sample() + unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9 + unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = + new Date(now - 180_001).toISOString() + const collect = vi.fn(async () => unhealthy) + const wait = vi.fn(async () => undefined) + await expect(runIncidentLivePreflight( + ['--state-file', stateFile(), '--retry-freshness'], + { now: () => now, collect, wait } + )).rejects.toThrow('cloud-monitoring/threshold_max') + expect(collect).toHaveBeenCalledOnce() + expect(wait).not.toHaveBeenCalled() + }) + + it('fails closed after the bounded freshness retry window', async () => { + const stale = sample() + stale.sources['cloud-monitoring']!.observedAt = new Date(now - 180_001).toISOString() + const collect = vi.fn(async () => stale) + const wait = vi.fn(async () => undefined) + await expect(runIncidentLivePreflight( + ['--state-file', stateFile(), '--retry-freshness'], + { now: () => now, collect, wait } + )).rejects.toThrow('cloud-monitoring/source_stale') + expect(collect).toHaveBeenCalledTimes(5) + expect(wait).toHaveBeenCalledTimes(4) + }) + + it('uses the supplied admin token without minting through gcloud', async () => { + const identityToken = vi.fn(async () => 'minted.token.value') + const gcloud = livePreflightGcloud( + { accessToken: async () => 'access-token', identityToken }, + { ORCA_RELAY_ADMIN_ID_TOKEN: 'supplied.token.value' } + ) + await expect(gcloud.identityToken!('audience')).resolves.toBe( + 'supplied.token.value' + ) + expect(identityToken).not.toHaveBeenCalled() + }) +}) diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts new file mode 100644 index 00000000000..e82627a3e80 --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts @@ -0,0 +1,195 @@ +import { readFile } from 'node:fs/promises' +import { resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { z } from 'zod' +import { createGcloudClient } from './gcloud-client.js' +import { suppliedIdentityToken } from './incident-monitor-cli.js' +import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js' +import { + evaluateIncidentSample, + preDrainDryRunPassed, + type IncidentSample +} from './incident-monitor.js' +import { createIncidentSampleCollector } from './incident-monitor-sources.js' + +const FRESHNESS_RETRY_ATTEMPTS = 5 +const FRESHNESS_RETRY_INTERVAL_MS = 15_000 +const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000 +// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave. +const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000 +const WAVE_INDEX_PATTERN = /^[0-3]$/ +const FRESHNESS_FAILURE_CODES = new Set([ + 'signal_missing', + 'signal_stale', + 'source_missing', + 'source_stale' +]) + +export function livePreflightGcloud( + gcloud: ReturnType, + environment: NodeJS.ProcessEnv = process.env +): ReturnType { + const token = suppliedIdentityToken(environment.ORCA_RELAY_ADMIN_ID_TOKEN) + return token ? { ...gcloud, identityToken: async () => token } : gcloud +} + +const PreflightStateSchema = z.object({ + schemaVersion: z.literal(4), + environment: z.literal('production'), + expectedSelector: AdmissionSelectorSchema, + migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']), + recoverySourceCellId: z.string().nullable(), + capacityCellId: z.string().nullable(), + preDrainDryRun: z.literal(true), + startedAt: z.string(), + windowStartedAt: z.string(), + durationMinutes: z.literal(15), + intervalMs: z.literal(60_000), + sampleCount: z.number().int().min(16), + lastSampleAt: z.string(), + frozenAt: z.null(), + completedAt: z.string() +}).superRefine((state, context) => { + const validRecovery = + state.migrationPolicy === 'recover-forward' && + state.capacityCellId === null && + state.recoverySourceCellId !== null && + state.expectedSelector.membership.existingOnly.includes( + state.recoverySourceCellId + ) + const validStrict = + state.migrationPolicy === 'strict' && + state.recoverySourceCellId === null && + state.capacityCellId === null + const validCapacity = + state.migrationPolicy === 'capacity-transition' && + state.recoverySourceCellId === null && + state.capacityCellId !== null && + state.expectedSelector.membership.general.includes(state.capacityCellId) + if (!validRecovery && !validStrict && !validCapacity) { + context.addIssue({ + code: 'custom', + message: 'relay live preflight migration policy is invalid' + }) + } +}) + +export async function runIncidentLivePreflight( + argv: string[], + dependencies: { + now?: () => number + wait?: (ms: number) => Promise + collect?: (expectedSelector: AdmissionSelector) => Promise + gcloud?: ReturnType + environment?: NodeJS.ProcessEnv + } = {} +): Promise { + const args = argv[0] === '--' ? argv.slice(1) : argv + const freshnessRetryCount = args.filter((arg) => arg === '--retry-freshness').length + const rest = args.filter((arg) => arg !== '--retry-freshness') + const stateArgs: string[] = [] + let waveIndex = '0' + let waveIndexCount = 0 + for (let index = 0; index < rest.length; index += 1) { + if (rest[index] === '--wave-index') { + waveIndexCount += 1 + waveIndex = rest[index + 1] ?? '' + index += 1 + } else { + stateArgs.push(rest[index] as string) + } + } + if ( + freshnessRetryCount > 1 || + waveIndexCount > 1 || + !WAVE_INDEX_PATTERN.test(waveIndex) || + stateArgs.length !== 2 || + stateArgs[0] !== '--state-file' || + !stateArgs[1] + ) { + throw new Error( + 'usage: --state-file [--wave-index <0-3>] [--retry-freshness]' + ) + } + const state = PreflightStateSchema.parse( + JSON.parse(await readFile(resolve(stateArgs[1]), 'utf8')) + ) + const now = dependencies.now ?? Date.now + const completedAt = Date.parse(state.completedAt) + const windowStartedAt = Date.parse(state.windowStartedAt) + const lastSampleAt = Date.parse(state.lastSampleAt) + const evidenceAgeMs = now() - completedAt + // Later same-cap waves start after sequential predecessor cell rolls, so the + // freshness bound grows by one cell-job timeout per predecessor; the live + // samples collected below still hold every wave to current health. + const maxEvidenceAgeMs = + MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS + if ( + !preDrainDryRunPassed(state) || + !Number.isFinite(windowStartedAt) || + completedAt - windowStartedAt < 15 * 60_000 || + !Number.isFinite(lastSampleAt) || + lastSampleAt > completedAt || + completedAt - lastSampleAt > state.intervalMs || + !Number.isFinite(completedAt) || + evidenceAgeMs < 0 || + evidenceAgeMs > maxEvidenceAgeMs + ) { + throw new Error('relay live preflight monitor evidence is incomplete or stale') + } + const gcloud = livePreflightGcloud( + dependencies.gcloud ?? createGcloudClient(), + dependencies.environment + ) + // Each predecessor same-cap apply wave reversibly isolates and restores its + // cell, advancing the selector generation by exactly 2 with membership + // unchanged (rollback is single-cell, so it never reaches a later wave), so + // the live selector comparison must expect the wave-adjusted generation. + const collectOptions = { + environment: state.environment, + expectedSelector: { + ...state.expectedSelector, + generation: state.expectedSelector.generation + 2 * Number(waveIndex) + }, + ...(dependencies.now ? { now: dependencies.now } : {}) + } + const injected = dependencies.collect + const collect = injected + ? () => injected(collectOptions.expectedSelector) + : createIncidentSampleCollector(gcloud, collectOptions) + const wait = dependencies.wait ?? ((ms: number) => new Promise((resolveWait) => { + setTimeout(resolveWait, ms) + })) + const attempts = freshnessRetryCount === 1 ? FRESHNESS_RETRY_ATTEMPTS : 1 + for (let attempt = 1; attempt <= attempts; attempt++) { + const evaluation = evaluateIncidentSample( + await collect(), + now(), + state.migrationPolicy, + state.recoverySourceCellId, + state.capacityCellId + ) + if (evaluation.status === 'green') return + const freshnessOnly = evaluation.failures.every((failure) => + FRESHNESS_FAILURE_CODES.has(failure.code) + ) + if (!freshnessOnly || attempt === attempts) { + throw new Error( + `relay live preflight failed: ${evaluation.failures + .map((failure) => `${failure.source}/${failure.code}`) + .join(',')}` + ) + } + console.warn( + `relay live preflight awaiting fresh evidence (${attempt}/${attempts - 1})` + ) + await wait(FRESHNESS_RETRY_INTERVAL_MS) + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runIncidentLivePreflight(process.argv.slice(2)).catch((error: unknown) => { + console.error(error instanceof Error ? error.message : 'relay live preflight failed') + process.exitCode = 1 + }) +} diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts new file mode 100644 index 00000000000..3e1f20a3cbb --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts @@ -0,0 +1,454 @@ +import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + parseIncidentMonitorArguments, + runIncidentMonitorCli +} from './incident-monitor-cli.js' +import type { IncidentSample } from './incident-monitor.js' +import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' + +const directories: string[] = [] +const startedAt = Date.parse('2026-07-28T00:00:00.000Z') +const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map((cell) => cell.cellId) +const selector = { + generation: 1, + membership: { + existingOnly: productionCells.slice(1), + migrationOnly: [], + general: [productionCells[0]!] + } +} +const selectorArguments = [ + '--expected-selector-generation', + '1', + '--expected-existing-only-cells', + productionCells.slice(1).join(','), + '--expected-migration-only-cells', + 'none', + '--expected-general-cells', + productionCells[0]! +] +const signal = (value: number, at: number) => ({ + value, + observedAt: new Date(at).toISOString() +}) + +function sample(at: number): IncidentSample { + const observedAt = new Date(at).toISOString() + const cellId = 'production-gce-c1' + return { + collectedAt: observedAt, + selector, + expectedSelector: selector, + cells: [{ + cellId, + runtimeKnown: true, + powered: true, + expectedAdmissionState: 'general' + }], + sources: { + 'active-probe': { + observedAt, + signals: { + 'director.health': signal(1, at), + 'director.ready': signal(1, at), + 'director.latency_ms': signal(1, at), + 'auth.health': signal(1, at), + 'auth.ready': signal(1, at), + 'auth.latency_ms': signal(1, at), + [`cell.${cellId}.health`]: signal(1, at), + [`cell.${cellId}.ready`]: signal(1, at), + [`cell.${cellId}.latency_ms`]: signal(1, at) + } + }, + 'cloud-monitoring': { + observedAt, + signals: { + 'cloud_sql.cpu': signal(0.1, at), + 'cloud_sql.memory': signal(0.1, at), + 'cloud_sql.backends': signal(1, at), + 'cloud_sql.lock_waits': signal(0, at), + 'cloud_sql.deadlocks': signal(0, at), + 'director.instances': signal(5, at), + 'director.cpu': signal(0.1, at), + 'director.memory': signal(0.1, at), + 'director.concurrency': signal(1, at), + 'director.errors': signal(0, at), + 'auth.errors': signal(0, at) + } + }, + 'relay-logs': { + observedAt, + signals: { + 'relay.pool_waiting': signal(0, at), + 'relay.pool_wait_ms': signal(0, at), + 'relay.postgres_retries': signal(0, at), + 'relay.postgres_retry_exhausted': signal(0, at), + [`cell.${cellId}.connections`]: signal(1, at), + [`cell.${cellId}.queued_bytes`]: signal(0, at) + } + }, + 'director-admin': { + observedAt, + signals: { + [`cell.${cellId}.admission_state`]: signal(2, at), + [`cell.${cellId}.heartbeat_fresh`]: signal(1, at), + [`cell.${cellId}.heartbeat_age_ms`]: signal(1, at), + [`cell.${cellId}.migration_blocked`]: signal(0, at), + [`cell.${cellId}.migration_target_inactive`]: signal(0, at) + } + } + } + } +} + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('incident monitor CLI', () => { + it('requires an exact selector and rejects invalid membership', () => { + expect(() => parseIncidentMonitorArguments([])).toThrow( + '--expected-selector-generation' + ) + expect(() => + parseIncidentMonitorArguments([ + '--incident-id', + 'incident-1', + '--expected-selector-generation', + '1', + '--expected-existing-only-cells', + productionCells.slice(1).join(','), + '--expected-migration-only-cells', + 'none', + '--expected-general-cells', + 'production-gce-c99' + ]) + ).toThrow('every configured cell exactly once') + expect(() => + parseIncidentMonitorArguments([ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--interval-seconds', + '61' + ]) + ).toThrow('between 1 and 60') + expect(() => + parseIncidentMonitorArguments([ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--duration-minutes', + '14' + ]) + ).toThrow('between 15 and 90') + expect(() => + parseIncidentMonitorArguments([ + '--incident-id', + 'incident-1', + '--expected-selector-generation', + '0', + '--expected-existing-only-cells', + productionCells.slice(1).join(','), + '--expected-migration-only-cells', + productionCells[0]!, + '--expected-general-cells', + 'none' + ]) + ).toThrow('generation 0 cannot represent migration-only') + expect(() => + parseIncidentMonitorArguments([ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--migration-policy', + 'recover-forward', + '--pre-drain-dry-run' + ]) + ).toThrow('requires an existing-only --recovery-source-cell-id') + expect(() => + parseIncidentMonitorArguments([ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--migration-policy', + 'capacity-transition', + '--pre-drain-dry-run' + ]) + ).toThrow('requires a general --capacity-cell-id') + expect( + parseIncidentMonitorArguments([ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--migration-policy', + 'capacity-transition', + '--capacity-cell-id', + productionCells[0]!, + '--pre-drain-dry-run' + ]) + ).toMatchObject({ + migrationPolicy: 'capacity-transition', + capacityCellId: productionCells[0]!, + recoverySourceCellId: null + }) + }) + + it('writes private durable checkpoints for a green pre-drain dry run', async () => { + const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) + directories.push(directory) + let now = startedAt + const output: string[] = [] + const code = await runIncidentMonitorCli( + [ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--pre-drain-dry-run', + '--output-directory', + directory + ], + { + cwd: directory, + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => sample(now), + writeOutput: (value) => output.push(value) + } + ) + expect(code).toBe(0) + const statePath = join(directory, 'incident-1.state.json') + const summaryPath = join(directory, 'incident-1.summaries.jsonl') + const markdownPath = join(directory, 'incident-1.summary.md') + expect(statSync(statePath).mode & 0o077).toBe(0) + expect(statSync(summaryPath).mode & 0o077).toBe(0) + expect(statSync(markdownPath).mode & 0o077).toBe(0) + const summaries = readFileSync(summaryPath, 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + expect(summaries.map((entry) => entry.checkpointMinute)).toEqual([0, 5, 15]) + expect(output.join('')).not.toContain('token') + expect(readFileSync(markdownPath, 'utf8')).toContain( + '| 0 | 15 | green | 16 | none |' + ) + expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({ + completedAt: new Date(startedAt + 15 * 60_000).toISOString(), + frozenAt: null, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + sampleCount: 16 + }) + }) + + it('runs a recovery dry run without masking blocked migrations', async () => { + const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) + directories.push(directory) + let now = startedAt + const recoverySelector = { + generation: 1, + membership: { + existingOnly: ['production-gce-c1'], + migrationOnly: [], + general: productionCells.slice(1) + } + } + const recoverySample = (): IncidentSample => { + const current = sample(now) + current.selector = recoverySelector + current.expectedSelector = recoverySelector + current.cells[0]!.expectedAdmissionState = 'existing-only' + current.sources['director-admin']!.signals[ + 'cell.production-gce-c1.admission_state' + ] = signal(0, now) + current.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_target_inactive' + ] = signal(30, now) + return current + } + const args = [ + '--incident-id', + 'incident-1', + '--expected-selector-generation', + '1', + '--expected-existing-only-cells', + 'production-gce-c1', + '--expected-migration-only-cells', + 'none', + '--expected-general-cells', + productionCells.slice(1).join(','), + '--pre-drain-dry-run', + '--migration-policy', + 'recover-forward', + '--recovery-source-cell-id', + 'production-gce-c1', + '--output-directory', + directory + ] + const dependencies = { + cwd: directory, + now: () => now, + wait: async (ms: number) => { + now += ms + }, + collect: async () => recoverySample(), + writeOutput: () => {} + } + await expect(runIncidentMonitorCli(args, dependencies)).resolves.toBe(0) + expect( + JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8')) + ).toMatchObject({ + migrationPolicy: 'recover-forward', + recoverySourceCellId: 'production-gce-c1', + capacityCellId: null, + frozenAt: null + }) + }) + + it('fails a frozen pre-drain gate after its first sample', async () => { + const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) + directories.push(directory) + let now = startedAt + let collections = 0 + let waits = 0 + const code = await runIncidentMonitorCli( + [ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--pre-drain-dry-run', + '--output-directory', + directory + ], + { + cwd: directory, + now: () => now, + wait: async (ms) => { + waits++ + now += ms + }, + collect: async () => { + collections++ + const unhealthy = sample(now) + unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] = + signal(801, now) + return unhealthy + }, + writeOutput: () => {} + } + ) + expect(code).toBe(2) + expect(collections).toBe(1) + expect(waits).toBe(0) + expect( + JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8')) + ).toMatchObject({ + completedAt: new Date(startedAt).toISOString(), + frozenAt: new Date(startedAt).toISOString(), + sampleCount: 1 + }) + }) + + it('requires --restart and preserves a latched freeze', async () => { + const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) + directories.push(directory) + let now = startedAt + const args = [ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--duration-minutes', + '15', + '--output-directory', + directory + ] + await runIncidentMonitorCli(args, { + cwd: directory, + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + const unhealthy = sample(now) + unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(801, now) + return unhealthy + }, + writeOutput: () => {} + }) + await expect( + runIncidentMonitorCli(args, { + cwd: directory, + now: () => now, + collect: async () => sample(now), + writeOutput: () => {} + }) + ).rejects.toThrow('pass --restart') + const changedAdmission = [...args] + changedAdmission[changedAdmission.indexOf('--expected-selector-generation') + 1] = '2' + await expect( + runIncidentMonitorCli([...changedAdmission, '--restart'], { + cwd: directory, + now: () => now, + collect: async () => sample(now), + writeOutput: () => {} + }) + ).rejects.toThrow('do not match') + await expect( + runIncidentMonitorCli([...args, '--restart'], { + cwd: directory, + now: () => now, + collect: async () => sample(now), + writeOutput: () => {} + }) + ).resolves.toBe(2) + }) + + it('resumes a gracefully segmented monitor without resetting continuity', async () => { + const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) + directories.push(directory) + let now = startedAt + const args = [ + '--incident-id', + 'incident-1', + ...selectorArguments, + '--duration-minutes', + '15', + '--output-directory', + directory + ] + const dependencies = { + cwd: directory, + now: () => now, + wait: async (ms: number) => { + now += ms + }, + collect: async () => sample(now), + writeOutput: () => {} + } + await expect( + runIncidentMonitorCli([...args, '--max-samples-this-run', '2'], dependencies) + ).resolves.toBe(0) + const statePath = join(directory, 'incident-1.state.json') + expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({ + completedAt: null, + sampleCount: 2, + lastSampleAt: new Date(startedAt + 60_000).toISOString() + }) + await expect( + runIncidentMonitorCli([...args, '--restart'], dependencies) + ).resolves.toBe(0) + expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({ + completedAt: new Date(startedAt + 15 * 60_000).toISOString(), + windowSequence: 0, + sampleCount: 17 + }) + }) +}) diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.ts new file mode 100644 index 00000000000..e090be7ea58 --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-monitor-cli.ts @@ -0,0 +1,493 @@ +import { randomUUID } from 'node:crypto' +import { + appendFile, + chmod, + mkdir, + open, + readFile, + rename, + stat, + writeFile +} from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { z } from 'zod' +import { relayOpsEnvironment } from './environment-config.js' +import { createGcloudClient } from './gcloud-client.js' +import { + AdmissionSelectorSchema, + normalizeSelectorMembership, + type AdmissionSelector +} from './incident-selector.js' +import { + initialIncidentMonitorState, + preDrainDryRunPassed, + runIncidentMonitor, + type IncidentCheckpoint, + type IncidentSample, + type IncidentMonitorState +} from './incident-monitor.js' +import { createIncidentSampleCollector } from './incident-monitor-sources.js' + +const StateSchema = z.object({ + schemaVersion: z.literal(4), + incidentId: z.string(), + environment: z.enum(['production', 'staging']), + expectedSelector: AdmissionSelectorSchema, + preDrainDryRun: z.boolean(), + migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']), + recoverySourceCellId: z.string().nullable(), + capacityCellId: z.string().nullable(), + startedAt: z.string(), + windowStartedAt: z.string().nullable(), + windowSequence: z.number().int().nonnegative(), + durationMinutes: z.number().int(), + intervalMs: z.number().int(), + nextCheckpointIndex: z.number().int().nonnegative(), + sampleCount: z.number().int().nonnegative(), + totalSampleCount: z.number().int().nonnegative(), + lastSampleAt: z.string().nullable(), + continuityEvents: z.array(z.object({ + recordedAt: z.string(), + windowSequence: z.number().int().nonnegative(), + failures: z.array(z.object({ + code: z.string(), + source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']), + signal: z.string().optional(), + observed: z.number().optional(), + threshold: z.number().optional() + })) + })), + frozenAt: z.string().nullable(), + failures: z.array(z.object({ + code: z.string(), + source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']), + signal: z.string().optional(), + observed: z.number().optional(), + threshold: z.number().optional() + })), + completedAt: z.string().nullable() +}) + +type CliOptions = { + environment: 'production' | 'staging' + incidentId: string + durationMinutes: number + intervalMs: number + expectedSelector: AdmissionSelector + stateFile: string + summaryFile: string + markdownFile: string + restart: boolean + preDrainDryRun: boolean + migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition' + recoverySourceCellId: string | null + capacityCellId: string | null + maxSamplesThisRun: number | null +} + +function parsePositiveInteger(value: string | undefined, name: string): number { + const parsed = Number(value) + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`) + } + return parsed +} + +export function parseIncidentMonitorArguments(argv: string[], cwd = process.cwd()): CliOptions { + const flags = new Set(['restart', 'pre-drain-dry-run']) + const valueArguments = new Set([ + 'duration-minutes', + 'environment', + 'expected-selector-generation', + 'expected-existing-only-cells', + 'expected-migration-only-cells', + 'expected-general-cells', + 'incident-id', + 'interval-seconds', + 'max-samples-this-run', + 'migration-policy', + 'output-directory', + 'recovery-source-cell-id', + 'capacity-cell-id' + ]) + const values: Record = {} + const enabledFlags = new Set() + for (let index = 0; index < argv.length; index++) { + const argument = argv[index] + if (!argument?.startsWith('--')) throw new Error(`invalid argument ${argument ?? ''}`) + const name = argument.slice(2) + if (flags.has(name)) { + enabledFlags.add(name) + continue + } + if (!valueArguments.has(name)) throw new Error(`unknown argument --${name}`) + const value = argv[++index] + if (!value || value.startsWith('--')) throw new Error(`missing --${name} value`) + values[name] = value + } + const environment = z.enum(['production', 'staging']).parse( + values.environment ?? 'production' + ) + const incidentId = values['incident-id'] ?? randomUUID() + if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{7,127}$/.test(incidentId)) { + throw new Error('--incident-id must be 8-128 safe characters') + } + const selectorGeneration = Number(values['expected-selector-generation']) + if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) { + throw new Error('--expected-selector-generation must be a nonnegative integer') + } + const configuredCells = new Set( + relayOpsEnvironment(environment).cells.map((cell) => cell.cellId) + ) + const cellList = (name: string): string[] => { + const value = values[name] + if (value === undefined) throw new Error(`--${name} is required; use none for an empty set`) + return value === 'none' ? [] : value.split(',').map((cellId) => cellId.trim()) + } + const expectedSelector = { + generation: selectorGeneration, + membership: normalizeSelectorMembership( + { + existingOnly: cellList('expected-existing-only-cells'), + migrationOnly: cellList('expected-migration-only-cells'), + general: cellList('expected-general-cells') + }, + configuredCells + ) + } + if (selectorGeneration === 0 && expectedSelector.membership.migrationOnly.length > 0) { + throw new Error('generation 0 cannot represent migration-only admission') + } + const preDrainDryRun = enabledFlags.has('pre-drain-dry-run') + const migrationPolicy = z.enum([ + 'strict', + 'recover-forward', + 'capacity-transition' + ]).parse( + values['migration-policy'] ?? 'strict' + ) + const recoverySourceCellId = + values['recovery-source-cell-id'] === undefined || + values['recovery-source-cell-id'] === 'none' + ? null + : values['recovery-source-cell-id'] + const capacityCellId = + values['capacity-cell-id'] === undefined || values['capacity-cell-id'] === 'none' + ? null + : values['capacity-cell-id'] + const durationMinutes = parsePositiveInteger( + values['duration-minutes'] ?? (preDrainDryRun ? '15' : '90'), + '--duration-minutes' + ) + const intervalMs = + parsePositiveInteger(values['interval-seconds'] ?? '60', '--interval-seconds') * 1_000 + if (durationMinutes < 15 || durationMinutes > 90) { + throw new Error('--duration-minutes must be between 15 and 90') + } + if (intervalMs > 60_000) { + throw new Error('--interval-seconds must be between 1 and 60') + } + if (preDrainDryRun && durationMinutes !== 15) { + throw new Error('--pre-drain-dry-run requires --duration-minutes 15') + } + if (migrationPolicy !== 'strict' && !preDrainDryRun) { + throw new Error(`--migration-policy ${migrationPolicy} requires --pre-drain-dry-run`) + } + if ( + migrationPolicy === 'recover-forward' && + ( + recoverySourceCellId === null || + !configuredCells.has(recoverySourceCellId) || + !expectedSelector.membership.existingOnly.includes(recoverySourceCellId) + ) + ) { + throw new Error( + '--migration-policy recover-forward requires an existing-only --recovery-source-cell-id' + ) + } + if (migrationPolicy !== 'recover-forward' && recoverySourceCellId !== null) { + throw new Error('--recovery-source-cell-id requires --migration-policy recover-forward') + } + if ( + migrationPolicy === 'capacity-transition' && + ( + capacityCellId === null || + !configuredCells.has(capacityCellId) || + !expectedSelector.membership.general.includes(capacityCellId) + ) + ) { + throw new Error( + '--migration-policy capacity-transition requires a general --capacity-cell-id' + ) + } + if (migrationPolicy !== 'capacity-transition' && capacityCellId !== null) { + throw new Error('--capacity-cell-id requires --migration-policy capacity-transition') + } + const directory = resolve(cwd, values['output-directory'] ?? '.relay-incidents') + const maxSamplesThisRun = values['max-samples-this-run'] + ? parsePositiveInteger(values['max-samples-this-run'], '--max-samples-this-run') + : null + return { + environment, + incidentId, + durationMinutes, + intervalMs, + expectedSelector, + stateFile: resolve(directory, `${incidentId}.state.json`), + summaryFile: resolve(directory, `${incidentId}.summaries.jsonl`), + markdownFile: resolve(directory, `${incidentId}.summary.md`), + restart: enabledFlags.has('restart'), + preDrainDryRun, + migrationPolicy, + recoverySourceCellId, + capacityCellId, + maxSamplesThisRun + } +} + +async function fileExists(path: string): Promise { + try { + await stat(path) + return true + } catch { + return false + } +} + +async function syncFile(path: string): Promise { + const handle = await open(path, 'r') + try { + await handle.sync() + } finally { + await handle.close() + } +} + +async function persistState(path: string, state: IncidentMonitorState): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }) + await chmod(dirname(path), 0o700) + const temporaryPath = `${path}.tmp` + await writeFile(temporaryPath, `${JSON.stringify(state)}\n`, { mode: 0o600 }) + await chmod(temporaryPath, 0o600) + await syncFile(temporaryPath) + await rename(temporaryPath, path) + await syncFile(path) +} + +async function appendCheckpoint(path: string, checkpoint: IncidentCheckpoint): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }) + await chmod(dirname(path), 0o700) + if (await fileExists(path)) { + const existing = (await readFile(path, 'utf8')) + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as IncidentCheckpoint) + if ( + existing.some((entry) => + entry.windowSequence === checkpoint.windowSequence && + entry.checkpointMinute === checkpoint.checkpointMinute + ) + ) return + } + await appendFile(path, `${JSON.stringify(checkpoint)}\n`, { mode: 0o600 }) + await chmod(path, 0o600) + await syncFile(path) +} + +function markdownFailure(checkpoint: IncidentCheckpoint): string { + if (checkpoint.failures.length === 0) return 'none' + return checkpoint.failures + .map((failure) => { + const signal = failure.signal ? `/${failure.signal}` : '' + return `${failure.source}/${failure.code}${signal}` + }) + .join(', ') +} + +async function writeMarkdownSummary( + path: string, + summaryPath: string, + incidentId: string, + environment: string +): Promise { + const checkpoints = (await readFile(summaryPath, 'utf8')) + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as IncidentCheckpoint) + const rows = checkpoints.map((checkpoint) => [ + `| ${checkpoint.windowSequence}`, + checkpoint.checkpointMinute, + checkpoint.status, + checkpoint.sampleCount, + `${markdownFailure(checkpoint)} |` + ].join(' | ')) + const markdown = [ + '# Relay incident monitor', + '', + `Incident: \`${incidentId}\``, + '', + `Environment: \`${environment}\``, + '', + `Expected selector: \`${JSON.stringify(checkpoints[0]?.expectedSelector ?? null)}\``, + '', + `Migration policy: \`${checkpoints[0]?.migrationPolicy ?? 'unknown'}\``, + '', + `Recovery source: \`${checkpoints[0]?.recoverySourceCellId ?? 'none'}\``, + '', + `Capacity cell: \`${checkpoints[0]?.capacityCellId ?? 'none'}\``, + '', + '| Window | Minute | Status | Samples | Failures |', + '| ---: | ---: | --- | ---: | --- |', + ...rows, + '' + ].join('\n') + const temporaryPath = `${path}.tmp` + await writeFile(temporaryPath, markdown, { mode: 0o600 }) + await chmod(temporaryPath, 0o600) + await syncFile(temporaryPath) + await rename(temporaryPath, path) + await syncFile(path) +} + +export function suppliedIdentityToken(value: string | undefined): string | null { + if (value === undefined) return null + if ( + value.length > 8_192 || + !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(value) + ) { + throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is invalid') + } + return value +} + +async function readInitialState( + options: CliOptions, + now: () => number = Date.now +): Promise { + const exists = await fileExists(options.stateFile) + if (exists && !options.restart) { + throw new Error('incident state already exists; pass --restart to resume it') + } + if (!exists && options.restart) throw new Error('no incident state exists to restart') + if (!exists) { + return initialIncidentMonitorState({ + incidentId: options.incidentId, + environment: options.environment, + expectedSelector: options.expectedSelector, + preDrainDryRun: options.preDrainDryRun, + migrationPolicy: options.migrationPolicy, + recoverySourceCellId: options.recoverySourceCellId, + capacityCellId: options.capacityCellId, + startedAt: new Date(now()).toISOString(), + durationMinutes: options.durationMinutes, + intervalMs: options.intervalMs + }) + } + const state = StateSchema.parse( + JSON.parse(await readFile(options.stateFile, 'utf8')) + ) as IncidentMonitorState + if ( + state.incidentId !== options.incidentId || + state.environment !== options.environment || + JSON.stringify(state.expectedSelector) !== JSON.stringify(options.expectedSelector) || + state.preDrainDryRun !== options.preDrainDryRun || + state.migrationPolicy !== options.migrationPolicy || + state.recoverySourceCellId !== options.recoverySourceCellId || + state.capacityCellId !== options.capacityCellId || + state.durationMinutes !== options.durationMinutes || + state.intervalMs !== options.intervalMs + ) { + throw new Error('restart arguments do not match durable incident state') + } + return state +} + +export async function runIncidentMonitorCli( + argv: string[], + dependencies: { + cwd?: string + now?: () => number + wait?: (ms: number) => Promise + gcloud?: ReturnType + collect?: () => Promise + writeOutput?: (value: string) => void + environment?: NodeJS.ProcessEnv + } = {} +): Promise { + const options = parseIncidentMonitorArguments(argv, dependencies.cwd) + const state = await readInitialState(options, dependencies.now) + const baseGcloud = dependencies.gcloud ?? createGcloudClient() + const token = suppliedIdentityToken( + (dependencies.environment ?? process.env).ORCA_RELAY_ADMIN_ID_TOKEN + ) + await persistState(options.stateFile, state) + const gcloud = token + ? { ...baseGcloud, identityToken: async () => token } + : baseGcloud + const collect = + dependencies.collect ?? + createIncidentSampleCollector(gcloud, { + environment: options.environment, + expectedSelector: options.expectedSelector, + ...(dependencies.now ? { now: dependencies.now } : {}) + }) + let samplesThisRun = 0 + const segmentedCollect = async (): Promise => { + try { + return await collect() + } finally { + samplesThisRun++ + } + } + const wait = + dependencies.wait ?? + ((ms: number) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms))) + const segmentComplete = Symbol('segment-complete') + const output = dependencies.writeOutput ?? ((value) => process.stdout.write(`${value}\n`)) + let result: IncidentMonitorState + try { + result = await runIncidentMonitor(state, { + now: dependencies.now ?? Date.now, + wait: async (ms) => { + if ( + options.maxSamplesThisRun !== null && + samplesThisRun >= options.maxSamplesThisRun + ) { + throw segmentComplete + } + await wait(ms) + }, + collect: segmentedCollect, + persist: async (nextState) => await persistState(options.stateFile, nextState), + checkpoint: async (checkpoint) => { + await appendCheckpoint(options.summaryFile, checkpoint) + await writeMarkdownSummary( + options.markdownFile, + options.summaryFile, + options.incidentId, + options.environment + ) + output(JSON.stringify(checkpoint)) + } + }) + } catch (error) { + if (error !== segmentComplete) throw error + return 0 + } + if (options.preDrainDryRun && !preDrainDryRunPassed(result)) return 2 + return result.frozenAt ? 2 : 0 +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runIncidentMonitorCli(process.argv.slice(2)) + .then((code) => { + process.exitCode = code + }) + .catch((error: unknown) => { + console.error(error instanceof Error ? error.message : 'incident monitor failed') + process.exitCode = 1 + }) +} diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts new file mode 100644 index 00000000000..09b7b16fa45 --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts @@ -0,0 +1,418 @@ +import { describe, expect, it } from 'vitest' +import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' +import type { GcloudClient } from './gcloud-client.js' +import { effectiveAdmissionState } from './incident-selector.js' +import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js' +import { + directorSignals, + GOOGLE_METRICS, + readGoogleMetric, + readGoogleMetricWithEmptyRetry, + relayFiveMinuteDeltaSignal +} from './incident-monitor-sources.js' + +const now = Date.parse('2026-07-28T10:00:00.000Z') +const startAt = new Date(now - 5 * 60_000).toISOString() +const endAt = new Date(now).toISOString() +const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map( + ({ cellId }) => cellId +) + +describe('incident monitor sources', () => { + it('uses legacy booleans only at selector generation zero', () => { + const membership = { + existingOnly: ['c1'], + migrationOnly: [], + general: ['c2'] + } + expect( + effectiveAdmissionState({ generation: 0, membership }, true, 'c1') + ).toBe('general') + expect( + effectiveAdmissionState({ generation: 1, membership }, true, 'c1') + ).toBe('existing-only') + }) + + it('sums DELTA metrics across the window and scopes every target exactly', async () => { + const filters: string[] = [] + const fetchImpl: typeof fetch = async (input) => { + const url = new URL(String(input)) + filters.push(url.searchParams.get('filter') ?? '') + return Response.json({ + timeSeries: [ + { + points: [ + { + interval: { endTime: new Date(now - 120_000).toISOString() }, + value: { int64Value: '2' } + }, + { + interval: { endTime: new Date(now - 60_000).toISOString() }, + value: { int64Value: '3' } + } + ] + } + ] + }) + } + const environment = RELAY_OPS_ENVIRONMENTS.production + const directorErrors = GOOGLE_METRICS.find( + (definition) => definition.signal === 'director.errors' + )! + const deadlocks = GOOGLE_METRICS.find( + (definition) => definition.signal === 'cloud_sql.deadlocks' + )! + await expect( + readGoogleMetric( + environment, + directorErrors, + 'secret-access-token', + startAt, + endAt, + fetchImpl + ) + ).resolves.toEqual({ + value: 5, + observedAt: new Date(now - 60_000).toISOString() + }) + await readGoogleMetric( + environment, + deadlocks, + 'secret-access-token', + startAt, + endAt, + fetchImpl + ) + expect(filters[0]).toContain( + 'resource.label."service_name"="orca-cloud-relay"' + ) + expect(filters[0]).toContain('metric.label."response_code"!="503"') + expect(filters[1]).toContain( + 'resource.label."database_id"="onorca-cloud:orca-cloud-auth-db"' + ) + }) + + it('zero-fills an expired sparse lock-wait point', async () => { + let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + const fetchImpl: typeof fetch = async () => Response.json({ + timeSeries: [{ + points: [{ + interval: { endTime: new Date(pointAt).toISOString() }, + value: { int64Value: '1' } + }] + }] + }) + const definition = GOOGLE_METRICS.find( + ({ signal }) => signal === 'cloud_sql.lock_waits' + )! + await expect(readGoogleMetric( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + fetchImpl + )).resolves.toEqual({ + value: 1, + observedAt: new Date(pointAt).toISOString() + }) + await expect(readGoogleMetric( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + fetchImpl, + () => now + 3_000 + )).resolves.toEqual({ + value: 1, + observedAt: new Date(pointAt).toISOString() + }) + pointAt-- + await expect(readGoogleMetric( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + fetchImpl + )).resolves.toEqual({ value: 0, observedAt: endAt }) + }) + + it('freshens a sparse zero without masking a recent nonzero lock wait', async () => { + let value = 0 + const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + const readAt = now + 11_879 + const fetchImpl: typeof fetch = async () => Response.json({ + timeSeries: [{ + points: [{ + interval: { endTime: new Date(pointAt).toISOString() }, + value: { int64Value: String(value) } + }] + }] + }) + const definition = GOOGLE_METRICS.find( + ({ signal }) => signal === 'cloud_sql.lock_waits' + )! + + const sparseZero = await readGoogleMetric( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + fetchImpl, + () => readAt + ) + expect(sparseZero).toEqual({ + value: 0, + observedAt: new Date(readAt).toISOString() + }) + expect(readAt - pointAt).toBe(191_879) + expect(readAt - Date.parse(sparseZero!.observedAt)).toBe(0) + + value = 20 + await expect(readGoogleMetric( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + fetchImpl, + () => readAt + )).resolves.toEqual({ + value: 20, + observedAt: new Date(pointAt).toISOString() + }) + }) + + it('preserves a recent nonzero lock wait across staggered series', async () => { + const nonzeroAt = now - 179_000 + const zeroAt = now - 178_000 + const definition = GOOGLE_METRICS.find( + ({ signal }) => signal === 'cloud_sql.lock_waits' + )! + const metric = await readGoogleMetric( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + async () => Response.json({ + timeSeries: [ + { + points: [{ + interval: { endTime: new Date(nonzeroAt).toISOString() }, + value: { int64Value: '7' } + }] + }, + { + points: [{ + interval: { endTime: new Date(zeroAt).toISOString() }, + value: { int64Value: '0' } + }] + } + ] + }) + ) + + expect(metric).toEqual({ + value: 7, + observedAt: new Date(nonzeroAt).toISOString() + }) + + await expect(readGoogleMetric( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + async () => Response.json({ + timeSeries: [{ + points: [ + { + interval: { endTime: new Date(nonzeroAt).toISOString() }, + value: { int64Value: '7' } + }, + { + interval: { endTime: new Date(zeroAt).toISOString() }, + value: { int64Value: '0' } + } + ] + }] + }) + )).resolves.toEqual({ value: 0, observedAt: endAt }) + }) + + it('retries an empty required metric without weakening its value', async () => { + let calls = 0 + const waits: number[] = [] + const definition = GOOGLE_METRICS.find( + ({ signal }) => signal === 'cloud_sql.cpu' + )! + await expect(readGoogleMetricWithEmptyRetry( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + async () => Response.json(calls++ === 0 + ? { timeSeries: [] } + : { + timeSeries: [{ + points: [{ + interval: { endTime: new Date(now - 60_000).toISOString() }, + value: { doubleValue: 0.81 } + }] + }] + }), + () => now, + async (ms) => { waits.push(ms) } + )).resolves.toEqual({ + value: 0.81, + observedAt: new Date(now - 60_000).toISOString() + }) + expect(calls).toBe(2) + expect(waits).toEqual([2_000]) + }) + + it('still reports a required metric missing after bounded retries', async () => { + let calls = 0 + const waits: number[] = [] + const definition = GOOGLE_METRICS.find( + ({ signal }) => signal === 'director.concurrency' + )! + await expect(readGoogleMetricWithEmptyRetry( + RELAY_OPS_ENVIRONMENTS.production, + definition, + 'secret-access-token', + startAt, + endAt, + async () => { + calls++ + return Response.json({ timeSeries: [] }) + }, + () => now, + async (ms) => { waits.push(ms) } + )).resolves.toBeNull() + expect(calls).toBe(3) + expect(waits).toEqual([2_000, 2_000]) + }) + + it('timestamps sparse retry aggregates at query completion', () => { + expect(relayFiveMinuteDeltaSignal({ + available: true, + points: [ + { at: new Date(now - 22 * 60_000).toISOString(), value: 7 }, + { at: new Date(now - 4 * 60_000).toISOString(), value: 2 } + ] + }, endAt)).toEqual({ value: 2, observedAt: endAt }) + expect(relayFiveMinuteDeltaSignal({ + available: true, + points: [{ at: new Date(now - 22 * 60_000).toISOString(), value: 7 }] + }, endAt)).toEqual({ value: 0, observedAt: endAt }) + expect(relayFiveMinuteDeltaSignal({ + available: false, + points: [] + }, endAt)).toBeNull() + }) + + it('aggregates admin state without returning tokens or response identities', async () => { + const identityToken = 'secret.header.signature' + const sensitiveIdentity = 'user@example.test' + const gcloud: GcloudClient = { + accessToken: async () => 'unused', + identityToken: async () => identityToken + } + let activeRequests = 0 + let maximumActiveRequests = 0 + let requestCount = 0 + const fetchImpl: typeof fetch = async (_input, init) => { + requestCount++ + activeRequests++ + maximumActiveRequests = Math.max(maximumActiveRequests, activeRequests) + expect(new Headers(init?.headers).get('authorization')).toBe( + `Bearer ${identityToken}` + ) + const body = JSON.parse(String(init?.body)) as { + cellId?: string + sourceCellId?: string + targetCellId?: string + } + await Promise.resolve() + activeRequests-- + if (!body.cellId && !body.sourceCellId) { + return Response.json({ + selector: { + generation: 1, + membership: { + existingOnly: productionCells.slice(2), + migrationOnly: ['production-gce-c2'], + general: ['production-gce-c1'] + } + } + }) + } + if (body.cellId) { + return Response.json({ + status: { + // Selector-era monitoring must ignore this legacy compatibility bit. + enabled: body.cellId !== 'production-gce-c1', + connectionCapacity: { + hardCap: body.cellId === 'production-gce-c1' ? 1_000 : 600 + }, + runtime: { + lastHeartbeatAt: now - 1_000, + heartbeatFresh: true + }, + userId: sensitiveIdentity + } + }) + } + return Response.json({ + blocked: 0, + blockedExpiredUnregistered: + body.sourceCellId === 'production-gce-c1' && + body.targetCellId === 'production-gce-c2' + ? 1 + : 0, + registeredTargetInactive: 0, + userId: sensitiveIdentity + }) + } + const result = await directorSignals( + 'production', + { + generation: 1, + membership: { + existingOnly: productionCells.slice(2), + migrationOnly: ['production-gce-c2'], + general: ['production-gce-c1'] + } + }, + gcloud, + now, + fetchImpl + ) + expect(requestCount).toBe(productionCells.length * 2) + expect(maximumActiveRequests).toBe(1) + expect( + result.source.signals['cell.production-gce-c1.migration_blocked'] + ).toMatchObject({ value: 1 }) + expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).toMatchObject({ + expectedAdmissionState: 'general' + }) + expect( + result.source.signals['cell.production-gce-c1.admission_state'] + ).toMatchObject({ value: 2 }) + expect( + result.source.signals['cell.production-gce-c1.connection_hard_cap'] + ).toMatchObject({ value: 1_000 }) + expect( + result.source.signals['cell.production-gce-c2.admission_state'] + ).toMatchObject({ value: 1 }) + const serialized = JSON.stringify(result) + expect(serialized).not.toContain(identityToken) + expect(serialized).not.toContain(sensitiveIdentity) + }) +}) diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.ts new file mode 100644 index 00000000000..0b78c2c4f6b --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.ts @@ -0,0 +1,646 @@ +import { z } from 'zod' +import { buildDashboardSnapshot } from './dashboard-snapshot.js' +import type { + RelayOpsEnvironment, + RelayOpsEnvironmentId +} from './environment-config.js' +import { relayOpsEnvironment } from './environment-config.js' +import type { GcloudClient } from './gcloud-client.js' +import type { RelayMetricSnapshot } from './monitoring-snapshot.js' +import { + AdmissionSelectorSchema, + effectiveAdmissionState, + normalizeSelectorMembership, + selectorCellState, + type AdmissionSelector, +} from './incident-selector.js' +import { + INCIDENT_MONITOR_THRESHOLDS, + type IncidentSample, + type IncidentSignal, + type IncidentSource +} from './incident-monitor.js' + +const NumericSchema = z.union([z.number(), z.string()]) + .transform(Number) + .pipe(z.number().finite()) +const MonitoringPointSchema = z.object({ + interval: z.object({ endTime: z.string() }), + value: z.object({ + doubleValue: NumericSchema.optional(), + int64Value: NumericSchema.optional(), + distributionValue: z.object({ + mean: NumericSchema.optional(), + range: z.object({ max: NumericSchema.optional() }).optional() + }).optional() + }) +}) +const MonitoringResponseSchema = z.object({ + timeSeries: z.array(z.object({ points: z.array(MonitoringPointSchema) })).default([]), + nextPageToken: z.string().optional() +}) +const CellStatusSchema = z.object({ + status: z.object({ + enabled: z.boolean(), + connectionCapacity: z + .object({ hardCap: z.number().int().positive() }) + .nullable(), + runtime: z.object({ + lastHeartbeatAt: z.number(), + heartbeatFresh: z.boolean() + }).nullable() + }) +}) +const SelectorStatusSchema = z.object({ + selector: AdmissionSelectorSchema +}) +const MigrationStatusSchema = z.object({ + blocked: z.number().int().nonnegative(), + registeredTargetInactive: z.number().int().nonnegative(), + blockedExpiredUnregistered: z.number().int().nonnegative() +}) + +export type GoogleMetricDefinition = { + signal: string + type: string + resourceFilter: string + aggregation: 'latest-max' | 'latest-sum' | 'window-sum' + emptyIsZero?: boolean + zeroAfterMs?: number +} + +export const GOOGLE_METRICS: GoogleMetricDefinition[] = [ + { + signal: 'cloud_sql.cpu', + type: 'cloudsql.googleapis.com/database/cpu/utilization', + resourceFilter: 'resource.type="cloudsql_database"', + aggregation: 'latest-max' + }, + { + signal: 'cloud_sql.memory', + type: 'cloudsql.googleapis.com/database/memory/utilization', + resourceFilter: 'resource.type="cloudsql_database"', + aggregation: 'latest-max' + }, + { + signal: 'cloud_sql.backends', + type: 'cloudsql.googleapis.com/database/postgresql/num_backends', + resourceFilter: 'resource.type="cloudsql_database"', + aggregation: 'latest-sum' + }, + { + signal: 'cloud_sql.lock_waits', + type: 'cloudsql.googleapis.com/database/postgresql/backends_in_wait', + resourceFilter: + 'resource.type="cloudsql_database" AND metric.label."wait_event_type"="Lock"', + aggregation: 'latest-max', + emptyIsZero: true, + zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + }, + { + signal: 'cloud_sql.deadlocks', + type: 'cloudsql.googleapis.com/database/postgresql/deadlock_count', + resourceFilter: 'resource.type="cloudsql_database"', + aggregation: 'window-sum', + emptyIsZero: true + }, + { + signal: 'director.instances', + type: 'run.googleapis.com/container/instance_count', + resourceFilter: 'resource.type="cloud_run_revision"', + aggregation: 'latest-sum' + }, + { + signal: 'director.cpu', + type: 'run.googleapis.com/container/cpu/utilizations', + resourceFilter: 'resource.type="cloud_run_revision"', + aggregation: 'latest-max' + }, + { + signal: 'director.memory', + type: 'run.googleapis.com/container/memory/utilizations', + resourceFilter: 'resource.type="cloud_run_revision"', + aggregation: 'latest-max' + }, + { + signal: 'director.concurrency', + type: 'run.googleapis.com/container/max_request_concurrencies', + resourceFilter: + 'resource.type="cloud_run_revision" AND metric.label."state"="active"', + aggregation: 'latest-max' + }, + { + signal: 'director.errors', + type: 'run.googleapis.com/request_count', + resourceFilter: + 'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx" AND metric.label."response_code"!="503"', + aggregation: 'window-sum', + emptyIsZero: true + }, + { + signal: 'auth.errors', + type: 'run.googleapis.com/request_count', + resourceFilter: + 'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx"', + aggregation: 'window-sum', + emptyIsZero: true + } +] + +function pointValue(point: z.infer): number { + return ( + point.value.doubleValue ?? + point.value.int64Value ?? + point.value.distributionValue?.range?.max ?? + point.value.distributionValue?.mean ?? + 0 + ) +} + +function serviceFilter(signal: string, directorService: string, authService: string): string { + if (signal.startsWith('director.')) { + return `resource.label."service_name"="${directorService}"` + } + if (signal.startsWith('auth.')) { + return `resource.label."service_name"="${authService}"` + } + return '' +} + +function targetFilter( + definition: GoogleMetricDefinition, + environment: RelayOpsEnvironment +): string { + if (definition.signal.startsWith('cloud_sql.')) { + return `resource.label."database_id"="${environment.project}:${environment.sqlInstance}"` + } + return serviceFilter( + definition.signal, + environment.directorService, + environment.authService + ) +} + +async function googleJson( + fetchImpl: typeof fetch, + token: string, + url: URL | string, + init: RequestInit = {} +): Promise { + const response = await fetchImpl(url, { + ...init, + headers: { + authorization: `Bearer ${token}`, + ...(init.body ? { 'content-type': 'application/json' } : {}) + }, + signal: AbortSignal.timeout(30_000) + }) + if (!response.ok) throw new Error(`Google telemetry returned ${response.status}`) + return await response.json() +} + +export async function readGoogleMetric( + environment: RelayOpsEnvironment, + definition: GoogleMetricDefinition, + token: string, + startAt: string, + endAt: string, + fetchImpl: typeof fetch, + now: () => number = () => Date.parse(endAt) +): Promise { + const url = new URL( + `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries` + ) + const filters = [ + `metric.type="${definition.type}"`, + definition.resourceFilter, + targetFilter(definition, environment) + ].filter(Boolean) + url.searchParams.set('filter', filters.join(' AND ')) + url.searchParams.set('interval.startTime', startAt) + url.searchParams.set('interval.endTime', endAt) + url.searchParams.set('view', 'FULL') + url.searchParams.set('pageSize', '1000') + const parsed = MonitoringResponseSchema.parse( + await googleJson(fetchImpl, token, url) + ) + if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete') + const queryEndMs = Date.parse(endAt) + const readAtMs = Math.max(queryEndMs, now()) + const readAt = new Date(readAtMs).toISOString() + const points = parsed.timeSeries.flatMap((series) => series.points) + if (points.length === 0) { + return definition.emptyIsZero ? { value: 0, observedAt: readAt } : null + } + const zeroAfterMs = definition.zeroAfterMs + if (definition.emptyIsZero && zeroAfterMs !== undefined) { + const latestSeriesPoints = parsed.timeSeries.flatMap((series) => { + const seriesNewestAt = Math.max( + ...series.points.map((point) => Date.parse(point.interval.endTime)) + ) + return series.points.filter( + (point) => Date.parse(point.interval.endTime) === seriesNewestAt + ) + }) + const futurePoints = latestSeriesPoints.filter( + (point) => Date.parse(point.interval.endTime) > queryEndMs + ) + if (futurePoints.length > 0) { + return { + value: Math.max(...futurePoints.map(pointValue)), + observedAt: new Date(Math.max( + ...futurePoints.map((point) => Date.parse(point.interval.endTime)) + )).toISOString() + } + } + const recentNonzero = latestSeriesPoints.filter((point) => { + const pointAt = Date.parse(point.interval.endTime) + return pointValue(point) > 0 && queryEndMs - pointAt <= zeroAfterMs + }) + if (recentNonzero.length === 0) return { value: 0, observedAt: readAt } + return { + value: Math.max(...recentNonzero.map(pointValue)), + observedAt: new Date(Math.min( + ...recentNonzero.map((point) => Date.parse(point.interval.endTime)) + )).toISOString() + } + } + const newestAt = Math.max(...points.map((point) => Date.parse(point.interval.endTime))) + const selected = definition.aggregation === 'window-sum' + ? points + : points.filter((point) => Date.parse(point.interval.endTime) === newestAt) + const values = selected.map(pointValue) + const value = + definition.aggregation !== 'latest-max' + ? values.reduce((total, entry) => total + entry, 0) + : Math.max(...values) + return { + value, + observedAt: new Date(newestAt).toISOString() + } +} + +export async function readGoogleMetricWithEmptyRetry( + environment: RelayOpsEnvironment, + definition: GoogleMetricDefinition, + token: string, + startAt: string, + endAt: string, + fetchImpl: typeof fetch, + now: () => number = () => Date.parse(endAt), + wait: (ms: number) => Promise = async (ms) => + await new Promise((resolve) => setTimeout(resolve, ms)) +): Promise { + for (let attempt = 0; attempt < 3; attempt++) { + const signal = await readGoogleMetric( + environment, + definition, + token, + startAt, + endAt, + fetchImpl, + now + ) + if (signal !== null || definition.emptyIsZero) return signal + if (attempt < 2) await wait(2_000) + } + return null +} + +function addSignal( + signals: Record, + name: string, + value: number | null, + observedAt: string | null +): void { + if (value === null || observedAt === null) return + signals[name] = { value, observedAt } +} + +function endpointSignals( + snapshot: Awaited>, + nowAt: string +): IncidentSource { + const signals: Record = {} + const addEndpoint = ( + prefix: string, + endpoint: { health: boolean | null; ready: boolean | null; latencyMs: number | null }, + unavailableIsZero = false + ) => { + addSignal( + signals, + `${prefix}.health`, + endpoint.health === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.health), + nowAt + ) + addSignal( + signals, + `${prefix}.ready`, + endpoint.ready === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.ready), + nowAt + ) + addSignal(signals, `${prefix}.latency_ms`, endpoint.latencyMs, nowAt) + } + addEndpoint('director', snapshot.resources.directorEndpoint) + addEndpoint('auth', snapshot.resources.authEndpoint) + for (const cell of snapshot.resources.cells) { + addEndpoint(`cell.${cell.cellId}`, cell.endpoint, true) + } + return { observedAt: nowAt, signals } +} + +export function relayFiveMinuteDeltaSignal( + metric: Pick, + endAt: string +): IncidentSignal | null { + if (!metric.available) return null + const endMs = Date.parse(endAt) + if (!Number.isFinite(endMs)) throw new Error('Relay telemetry end time is invalid') + return { + value: metric.points + .filter((point) => { + const pointMs = Date.parse(point.at) + return pointMs >= endMs - 300_000 && pointMs <= endMs + }) + .reduce((total, point) => total + point.value, 0), + observedAt: endAt + } +} + +function relaySignals( + snapshot: Awaited> +): IncidentSource { + const metrics = snapshot.monitoring.metrics + const signals: Record = {} + addSignal( + signals, + 'relay.pool_waiting', + metrics.db_waiters_max.latest, + metrics.db_waiters_max.latestAt + ) + addSignal( + signals, + 'relay.pool_wait_ms', + metrics.db_wait_ms_max.latest, + metrics.db_wait_ms_max.latestAt + ) + const retries = relayFiveMinuteDeltaSignal( + metrics.postgres_retries, + snapshot.monitoring.endAt + ) + if (retries) signals['relay.postgres_retries'] = retries + const retryExhausted = relayFiveMinuteDeltaSignal( + metrics.postgres_retry_exhausted, + snapshot.monitoring.endAt + ) + if (retryExhausted) signals['relay.postgres_retry_exhausted'] = retryExhausted + for (const cell of snapshot.resources.cells) { + addSignal( + signals, + `cell.${cell.cellId}.connections`, + metrics.total_connections.latestByCell[cell.cellId] ?? null, + metrics.total_connections.latestAt + ) + addSignal( + signals, + `cell.${cell.cellId}.queued_bytes`, + metrics.queued_bytes.latestByCell[cell.cellId] ?? null, + metrics.queued_bytes.latestAt + ) + } + const observedTimes = Object.values(signals).map((entry) => Date.parse(entry.observedAt)) + if (observedTimes.length === 0) throw new Error('Relay telemetry is unavailable') + const observedAt = new Date(Math.max(...observedTimes)).toISOString() + return { observedAt, signals } +} + +async function adminPost( + fetchImpl: typeof fetch, + origin: string, + token: string, + path: string, + body: unknown +): Promise { + const response = await fetchImpl(`${origin}${path}`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }) + if (!response.ok) throw new Error(`Relay admin telemetry returned ${response.status}`) + return await response.json() +} + +export async function directorSignals( + environmentId: RelayOpsEnvironmentId, + expectedSelector: AdmissionSelector, + gcloud: GcloudClient, + nowMs: number, + fetchImpl: typeof fetch +): Promise<{ + source: IncidentSource + selector: AdmissionSelector + cells: IncidentSample['cells'] +}> { + const environment = relayOpsEnvironment(environmentId) + if (!gcloud.identityToken) throw new Error('gcloud identity-token support is unavailable') + const token = await gcloud.identityToken(`${environment.directorOrigin}/v1/admin/drain`) + const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId)) + const rawSelector = SelectorStatusSchema.parse( + await adminPost( + fetchImpl, + environment.directorOrigin, + token, + '/v1/admin/admission-selector/status', + { v: 1 } + ) + ).selector + const selector = { + generation: rawSelector.generation, + membership: normalizeSelectorMembership(rawSelector.membership, configuredCellIds) + } + const statuses: Array<{ + cell: RelayOpsEnvironment['cells'][number] + status: z.infer['status'] + }> = [] + for (const cell of environment.cells) { + statuses.push({ + cell, + status: CellStatusSchema.parse( + await adminPost(fetchImpl, environment.directorOrigin, token, '/v1/admin/cell-status', { + v: 1, + cellId: cell.cellId + }) + ).status + }) + } + const migrationEntries: Array<{ + sourceCellId: string + migration: z.infer + }> = [] + const migrationTargets = new Set(expectedSelector.membership.migrationOnly) + for (const source of environment.cells) { + for (const target of environment.cells) { + if (source.cellId === target.cellId || !migrationTargets.has(target.cellId)) continue + migrationEntries.push({ + sourceCellId: source.cellId, + migration: MigrationStatusSchema.parse( + await adminPost( + fetchImpl, + environment.directorOrigin, + token, + '/v1/admin/evacuation-status', + { + v: 1, + sourceCellId: source.cellId, + targetCellId: target.cellId, + completeReady: false + } + ) + ) + }) + } + } + const migrationBySource = new Map() + for (const { sourceCellId, migration } of migrationEntries) { + const aggregate = migrationBySource.get(sourceCellId) ?? { blocked: 0, targetInactive: 0 } + aggregate.blocked += migration.blocked + migration.blockedExpiredUnregistered + aggregate.targetInactive += migration.registeredTargetInactive + migrationBySource.set(sourceCellId, aggregate) + } + const nowAt = new Date(nowMs).toISOString() + const signals: Record = {} + for (const { cell, status } of statuses) { + const prefix = `cell.${cell.cellId}` + const admissionState = effectiveAdmissionState( + selector, + status.enabled, + cell.cellId + ) + addSignal( + signals, + `${prefix}.admission_state`, + ['existing-only', 'migration-only', 'general'].indexOf(admissionState), + nowAt + ) + addSignal( + signals, + `${prefix}.connection_hard_cap`, + status.connectionCapacity?.hardCap ?? + INCIDENT_MONITOR_THRESHOLDS.cellConnections, + nowAt + ) + if (status.runtime) { + addSignal( + signals, + `${prefix}.heartbeat_fresh`, + Number(status.runtime.heartbeatFresh), + nowAt + ) + addSignal( + signals, + `${prefix}.heartbeat_age_ms`, + Math.max(0, nowMs - status.runtime.lastHeartbeatAt), + nowAt + ) + } + const migration = migrationBySource.get(cell.cellId) ?? { blocked: 0, targetInactive: 0 } + addSignal(signals, `${prefix}.migration_blocked`, migration.blocked, nowAt) + addSignal( + signals, + `${prefix}.migration_target_inactive`, + migration.targetInactive, + nowAt + ) + } + return { + source: { observedAt: nowAt, signals }, + selector, + cells: statuses.map(({ cell }) => ({ + cellId: cell.cellId, + runtimeKnown: true, + powered: true, + expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId) + })) + } +} + +export type IncidentSampleCollectorOptions = { + environment: RelayOpsEnvironmentId + expectedSelector: AdmissionSelector + fetchImpl?: typeof fetch + now?: () => number +} + +export function createIncidentSampleCollector( + gcloud: GcloudClient, + options: IncidentSampleCollectorOptions +): () => Promise { + const fetchImpl = options.fetchImpl ?? fetch + const now = options.now ?? Date.now + return async () => { + const nowMs = now() + const nowAt = new Date(nowMs).toISOString() + const startAt = new Date(nowMs - 5 * 60_000).toISOString() + const environment = relayOpsEnvironment(options.environment) + const accessToken = gcloud.accessToken() + const cloudMetricEntries = accessToken.then(async (token) => await Promise.all( + GOOGLE_METRICS.map(async (definition) => [ + definition.signal, + await readGoogleMetricWithEmptyRetry( + environment, + definition, + token, + startAt, + nowAt, + fetchImpl, + now + ) + ] as const) + )) + const [snapshot, director, metricEntries] = await Promise.all([ + buildDashboardSnapshot(options.environment, gcloud, { + windowMinutes: 5, + now: new Date(nowMs), + fetchImpl + }), + directorSignals( + options.environment, + options.expectedSelector, + gcloud, + nowMs, + fetchImpl + ), + cloudMetricEntries + ]) + const cloudSignals = Object.fromEntries( + metricEntries.filter((entry): entry is [string, IncidentSignal] => entry[1] !== null) + ) + const relay = relaySignals(snapshot) + const poweredByCell = new Map( + snapshot.resources.cells.map((cell) => [ + cell.cellId, + { + runtimeKnown: cell.targetSize !== null, + powered: (cell.targetSize ?? 0) > 0 + } + ]) + ) + return { + collectedAt: nowAt, + selector: director.selector, + expectedSelector: options.expectedSelector, + sources: { + 'active-probe': endpointSignals(snapshot, nowAt), + 'cloud-monitoring': { observedAt: nowAt, signals: cloudSignals }, + 'relay-logs': relay, + 'director-admin': director.source + }, + cells: director.cells.map((cell) => ({ + ...cell, + runtimeKnown: poweredByCell.get(cell.cellId)?.runtimeKnown ?? false, + powered: poweredByCell.get(cell.cellId)?.powered ?? false + })) + } + } +} diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts new file mode 100644 index 00000000000..51153bb63e1 --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts @@ -0,0 +1,786 @@ +import { describe, expect, it } from 'vitest' +import { + evaluateIncidentSample, + INCIDENT_CHECKPOINT_MINUTES, + INCIDENT_MONITOR_THRESHOLDS, + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS, + initialIncidentMonitorState, + preDrainDryRunPassed, + runIncidentMonitor, + type IncidentSample +} from './incident-monitor.js' + +const startedAt = Date.parse('2026-07-28T00:00:00.000Z') +const selector = { + generation: 1, + membership: { + existingOnly: [], + migrationOnly: [], + general: ['production-gce-c1'] + } +} +const signal = (value: number, at = startedAt) => ({ + value, + observedAt: new Date(at).toISOString() +}) + +function healthySample(at = startedAt): IncidentSample { + const observedAt = new Date(at).toISOString() + return { + collectedAt: observedAt, + selector, + expectedSelector: selector, + cells: [{ + cellId: 'production-gce-c1', + runtimeKnown: true, + powered: true, + expectedAdmissionState: 'general' + }], + sources: { + 'active-probe': { + observedAt, + signals: { + 'director.health': signal(1, at), + 'director.ready': signal(1, at), + 'director.latency_ms': signal(100, at), + 'auth.health': signal(1, at), + 'auth.ready': signal(1, at), + 'auth.latency_ms': signal(100, at), + 'cell.production-gce-c1.health': signal(1, at), + 'cell.production-gce-c1.ready': signal(1, at), + 'cell.production-gce-c1.latency_ms': signal(100, at) + } + }, + 'cloud-monitoring': { + observedAt, + signals: { + 'cloud_sql.cpu': signal(0.2, at), + 'cloud_sql.memory': signal(0.3, at), + 'cloud_sql.backends': signal(12, at), + 'cloud_sql.lock_waits': signal(0, at), + 'cloud_sql.deadlocks': signal(0, at), + 'director.instances': signal(5, at), + 'director.cpu': signal(0.2, at), + 'director.memory': signal(0.3, at), + 'director.concurrency': signal(5, at), + 'director.errors': signal(0, at), + 'auth.errors': signal(0, at) + } + }, + 'relay-logs': { + observedAt, + signals: { + 'relay.pool_waiting': signal(0, at), + 'relay.pool_wait_ms': signal(1, at), + 'relay.postgres_retries': signal(0, at), + 'relay.postgres_retry_exhausted': signal(0, at), + 'cell.production-gce-c1.connections': signal(100, at), + 'cell.production-gce-c1.queued_bytes': signal(0, at) + } + }, + 'director-admin': { + observedAt, + signals: { + 'cell.production-gce-c1.admission_state': signal(2, at), + 'cell.production-gce-c1.heartbeat_fresh': signal(1, at), + 'cell.production-gce-c1.heartbeat_age_ms': signal(1_000, at), + 'cell.production-gce-c1.migration_blocked': signal(0, at), + 'cell.production-gce-c1.migration_target_inactive': signal(0, at) + } + } + } + } +} + +describe('incident monitor evaluator', () => { + it('accepts a complete fresh sample at every exact boundary', () => { + const sample = healthySample() + sample.sources['active-probe']!.signals['director.latency_ms'] = + signal(INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs) + sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = + signal(INCIDENT_MONITOR_THRESHOLDS.cloudSqlCpuUtilization) + sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = + signal(INCIDENT_MONITOR_THRESHOLDS.relayPoolWaitMs) + sample.sources['relay-logs']!.signals['relay.postgres_retries'] = + signal(INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries) + sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = + signal(INCIDENT_MONITOR_THRESHOLDS.cloudSqlBackends) + expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ + status: 'green', + failures: [] + }) + }) + + it('freezes when postgres retries exceed the recalibrated ceiling', () => { + const sample = healthySample() + sample.sources['relay-logs']!.signals['relay.postgres_retries'] = + signal(INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries + 1) + expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ + status: 'freeze', + failures: [ + expect.objectContaining({ signal: 'relay.postgres_retries', threshold: 300 }) + ] + }) + }) + + it('allows missing auth readiness and legacy existing-only connections', () => { + const sample = healthySample() + const legacySelector = { + generation: 1, + membership: { + existingOnly: ['production-gce-c1'], + migrationOnly: [], + general: [] + } + } + sample.selector = legacySelector + sample.expectedSelector = legacySelector + sample.cells[0]!.expectedAdmissionState = 'existing-only' + delete sample.sources['active-probe']!.signals['auth.ready'] + sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = + signal(900) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.admission_state' + ] = signal(0) + expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ + status: 'green', + failures: [] + }) + }) + + it('fails loudly on every missing or stale source', () => { + const missing = healthySample() + delete missing.sources['relay-logs'] + expect(evaluateIncidentSample(missing, startedAt).failures).toContainEqual({ + code: 'source_missing', + source: 'relay-logs' + }) + const stale = healthySample(startedAt - 180_001) + const failures = evaluateIncidentSample(stale, startedAt).failures + expect(failures.some((failure) => failure.source === 'cloud-monitoring')).toBe(true) + expect(failures.some((failure) => failure.source === 'active-probe')).toBe(true) + }) + + it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => { + const sample = healthySample() + sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81) + sample.sources['cloud-monitoring']!.signals['director.instances'] = signal(7) + sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(801) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.heartbeat_age_ms' + ] = signal(45_001) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_blocked' + ] = signal(1) + sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = + signal(501) + const evaluation = evaluateIncidentSample(sample, startedAt) + expect(evaluation.status).toBe('freeze') + expect(evaluation.failures.map((failure) => failure.signal)).toEqual( + expect.arrayContaining([ + 'cloud_sql.cpu', + 'director.instances', + 'relay.pool_waiting', + 'cell.production-gce-c1.connections', + 'cell.production-gce-c1.heartbeat_age_ms', + 'cell.production-gce-c1.migration_blocked' + ]) + ) + }) + + it('uses each cell reported physical connection cap', () => { + const sample = healthySample() + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.connection_hard_cap' + ] = signal(1_000) + sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = + signal(999) + + expect(evaluateIncidentSample(sample, startedAt).status).toBe('green') + sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = + signal(1_000) + expect(evaluateIncidentSample(sample, startedAt).status).toBe('freeze') + }) + + it('allows five active directors plus the warm rollback', () => { + const sample = healthySample() + sample.sources['cloud-monitoring']!.signals['director.instances'] = signal(6) + expect(evaluateIncidentSample(sample, startedAt).status).toBe('green') + }) + + it('allows bounded relay pool waiting below the latency ceiling', () => { + const sample = healthySample() + sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(800) + sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = signal(2_500) + expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ + status: 'green', + failures: [] + }) + sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = signal(2_501) + expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ + code: 'threshold_max', + source: 'relay-logs', + signal: 'relay.pool_wait_ms', + observed: 2_501, + threshold: 2_500 + }) + }) + + it('bounds Cloud SQL backends above measured healthy peaks', () => { + const sample = healthySample() + sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(250) + expect(evaluateIncidentSample(sample, startedAt).status).toBe('green') + sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(251) + expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ + code: 'threshold_max', + source: 'cloud-monitoring', + signal: 'cloud_sql.backends', + observed: 251, + threshold: 250 + }) + }) + + it('bounds SQL lock waiters and keeps deadlocks zero-tolerance', () => { + const sample = healthySample() + sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(20) + expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ + status: 'green', + failures: [] + }) + sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(21) + expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ + code: 'threshold_max', + source: 'cloud-monitoring', + signal: 'cloud_sql.lock_waits', + observed: 21, + threshold: 20 + }) + sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(0) + sample.sources['cloud-monitoring']!.signals['cloud_sql.deadlocks'] = signal(1) + expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ + code: 'threshold_max', + source: 'cloud-monitoring', + signal: 'cloud_sql.deadlocks', + observed: 1, + threshold: 0 + }) + }) + + it('allows only registered target inactivity during forward recovery', () => { + const sample = healthySample() + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_target_inactive' + ] = signal(40) + expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ + code: 'threshold_max', + source: 'director-admin', + signal: 'cell.production-gce-c1.migration_target_inactive', + observed: 40, + threshold: 0 + }) + expect( + evaluateIncidentSample( + sample, + startedAt, + 'recover-forward', + 'production-gce-c1' + ) + ).toMatchObject({ + status: 'green', + failures: [] + }) + expect( + evaluateIncidentSample( + sample, + startedAt, + 'recover-forward', + 'production-gce-c2' + ).failures + ).toContainEqual(expect.objectContaining({ + signal: 'cell.production-gce-c1.migration_target_inactive' + })) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_blocked' + ] = signal(1) + expect( + evaluateIncidentSample( + sample, + startedAt, + 'recover-forward', + 'production-gce-c1' + ).failures + ).toContainEqual({ + code: 'threshold_max', + source: 'director-admin', + signal: 'cell.production-gce-c1.migration_blocked', + observed: 1, + threshold: 0 + }) + }) + + it('scopes registered target inactivity to the capacity cell', () => { + const sample = healthySample() + const scopedSelector = { + generation: 1, + membership: { + existingOnly: ['production-gce-c1'], + migrationOnly: [], + general: ['production-gce-c2', 'production-gce-c3'] + } + } + sample.selector = scopedSelector + sample.expectedSelector = scopedSelector + sample.cells[0]!.expectedAdmissionState = 'existing-only' + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.admission_state' + ] = signal(0) + sample.cells.push({ + cellId: 'production-gce-c2', + runtimeKnown: true, + powered: true, + expectedAdmissionState: 'general' + }) + sample.cells.push({ + cellId: 'production-gce-c3', + runtimeKnown: true, + powered: true, + expectedAdmissionState: 'general' + }) + for (const sourceName of ['active-probe', 'relay-logs', 'director-admin'] as const) { + const signals = sample.sources[sourceName]!.signals + for (const [name, value] of Object.entries(signals)) { + if (name.includes('production-gce-c1')) { + for (const cellId of ['production-gce-c2', 'production-gce-c3']) { + signals[name.replace('production-gce-c1', cellId)] = value + } + } + } + } + for (const cellId of ['production-gce-c2', 'production-gce-c3']) { + sample.sources['director-admin']!.signals[ + `cell.${cellId}.admission_state` + ] = signal(2) + } + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.migration_target_inactive' + ] = signal(40) + expect( + evaluateIncidentSample( + sample, + startedAt, + 'capacity-transition', + null, + 'production-gce-c2' + ) + ).toMatchObject({ status: 'green', failures: [] }) + expect( + evaluateIncidentSample( + sample, + startedAt, + 'capacity-transition', + null, + 'production-gce-c3' + ) + ).toMatchObject({ status: 'green', failures: [] }) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c2.migration_target_inactive' + ] = signal(1) + expect( + evaluateIncidentSample( + sample, + startedAt, + 'capacity-transition', + null, + 'production-gce-c2' + ).failures + ).toContainEqual(expect.objectContaining({ + signal: 'cell.production-gce-c2.migration_target_inactive' + })) + }) + + it('freezes when expected admission has no powered runtime', () => { + const sample = healthySample() + sample.cells[0]!.powered = false + const evaluation = evaluateIncidentSample(sample, startedAt) + expect(evaluation.failures).toContainEqual({ + code: 'expected_admission_without_runtime', + source: 'director-admin', + signal: 'cell.production-gce-c1.powered', + observed: 0, + threshold: 1 + }) + }) + + it('ignores stale runtime signals for an expected offline existing-only cell', () => { + const sample = healthySample() + sample.cells[0] = { + ...sample.cells[0]!, + powered: false, + expectedAdmissionState: 'existing-only' + } + sample.expectedSelector = { + generation: sample.expectedSelector.generation, + membership: { + existingOnly: ['production-gce-c1'], + migrationOnly: [], + general: [] + } + } + sample.selector = sample.expectedSelector + sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] = signal(0) + sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.admission_state' + ] = signal(0) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.heartbeat_fresh' + ] = signal(0) + sample.sources['director-admin']!.signals[ + 'cell.production-gce-c1.heartbeat_age_ms' + ] = signal(9_000_000) + + expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ + status: 'green', + failures: [] + }) + }) + + it('freezes when cell power inventory is unavailable', () => { + const sample = healthySample() + sample.cells[0]!.runtimeKnown = false + expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ + code: 'runtime_power_unknown', + source: 'cloud-monitoring', + signal: 'cell.production-gce-c1.powered' + }) + }) + + it('freezes on selector generation or tri-state membership drift', () => { + const generation = healthySample() + generation.selector = { ...generation.selector, generation: 2 } + expect(evaluateIncidentSample(generation, startedAt).failures).toContainEqual( + expect.objectContaining({ code: 'selector_mismatch' }) + ) + + const membership = healthySample() + membership.selector = { + generation: 1, + membership: { + existingOnly: ['production-gce-c1'], + migrationOnly: [], + general: [] + } + } + expect(evaluateIncidentSample(membership, startedAt).failures).toContainEqual( + expect.objectContaining({ code: 'selector_mismatch' }) + ) + }) +}) + +describe('incident monitor lifecycle', () => { + it('persists the exact 90-minute checkpoints while polling every minute', async () => { + let now = startedAt + const checkpoints: number[] = [] + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: false, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 90, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => healthySample(now), + persist: async () => {}, + checkpoint: async (summary) => { + checkpoints.push(summary.checkpointMinute) + } + }) + expect(checkpoints).toEqual([...INCIDENT_CHECKPOINT_MINUTES]) + expect(result.sampleCount).toBe(91) + expect(result.completedAt).not.toBeNull() + expect(result.frozenAt).toBeNull() + }) + + it('latches monitor freeze across a restart without rewriting its time', async () => { + let now = startedAt + let unhealthy = true + let persisted = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: false, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const stop = new Error('stop after first persistence') + await expect( + runIncidentMonitor(persisted, { + now: () => now, + wait: async () => { + throw stop + }, + collect: async () => { + const sample = healthySample(now) + if (unhealthy) { + sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(31, now) + } + return sample + }, + persist: async (state) => { + persisted = structuredClone(state) + }, + checkpoint: async () => {} + }) + ).rejects.toThrow('stop after first persistence') + const frozenAt = persisted.frozenAt + unhealthy = false + now += 60_000 + const result = await runIncidentMonitor(persisted, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => healthySample(now), + persist: async () => {}, + checkpoint: async () => {} + }) + expect(result.frozenAt).toBe(frozenAt) + expect(preDrainDryRunPassed(result)).toBe(false) + }) + + it.each([15, 90])( + 'restarts a %i-minute continuous window after stale telemetry', + async (durationMinutes) => { + let now = startedAt + let staleInjected = false + const checkpoints: Array<[number, number]> = [] + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: durationMinutes === 15, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + if (!staleInjected && now === startedAt + 5 * 60_000) { + staleInjected = true + return healthySample(now - 180_001) + } + return healthySample(now) + }, + persist: async () => {}, + checkpoint: async (summary) => { + checkpoints.push([summary.windowSequence, summary.checkpointMinute]) + } + }) + expect(result.windowSequence).toBe(1) + expect(result.windowStartedAt).toBe( + new Date(startedAt + 6 * 60_000).toISOString() + ) + expect(result.completedAt).toBe( + new Date(startedAt + (durationMinutes + 6) * 60_000).toISOString() + ) + expect(result.sampleCount).toBe(durationMinutes + 1) + expect(result.continuityEvents).toHaveLength(1) + expect(result.continuityEvents[0]!.failures).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'source_stale' }) + ]) + ) + expect(checkpoints).toContainEqual([1, durationMinutes]) + expect(result.frozenAt).toBeNull() + } + ) + + it('resets at the next fresh sample after a runner gap', async () => { + let now = startedAt + 10 * 60_000 + const state = { + ...initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }), + lastSampleAt: new Date(startedAt).toISOString(), + sampleCount: 1, + totalSampleCount: 1 + } + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => healthySample(now), + persist: async () => {}, + checkpoint: async () => {} + }) + expect(result.windowSequence).toBe(1) + expect(result.windowStartedAt).toBe(new Date(startedAt + 10 * 60_000).toISOString()) + expect(result.continuityEvents[0]!.failures[0]!.code).toBe('monitor_gap') + expect(result.sampleCount).toBe(16) + }) + + it('fails a dry run after 25 total minutes of continuity resets', async () => { + let now = startedAt + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => + healthySample(now === startedAt + 10 * 60_000 ? now - 180_001 : now), + persist: async () => {}, + checkpoint: async () => {} + }) + + expect(result.completedAt).toBe( + new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS).toISOString() + ) + expect(result.frozenAt).not.toBeNull() + expect(result.windowSequence).toBe(1) + expect(result.sampleCount).toBe(15) + expect(result.failures).toContainEqual({ + code: 'continuity_deadline_exceeded', + source: 'active-probe', + observed: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS, + threshold: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + }) + expect(preDrainDryRunPassed(result)).toBe(false) + }) + + it('fails an overdue resumed dry run before collecting again', async () => { + const now = startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + 1 + let collections = 0 + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async () => {}, + collect: async () => { + collections++ + return healthySample(now) + }, + persist: async () => {}, + checkpoint: async () => {} + }) + + expect(collections).toBe(0) + expect(result.failures).toContainEqual(expect.objectContaining({ + code: 'continuity_deadline_exceeded' + })) + }) + + it('requires a completed green 15-minute dry run', () => { + const state = { + ...initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }), + sampleCount: 16, + completedAt: new Date(startedAt + 15 * 60_000).toISOString() + } + expect(preDrainDryRunPassed(state)).toBe(true) + expect(preDrainDryRunPassed({ ...state, frozenAt: state.startedAt })).toBe(false) + expect(preDrainDryRunPassed({ + ...state, + completedAt: new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + 1).toISOString() + })).toBe(false) + }) + + it('keeps poll starts on the configured cadence after collection time', async () => { + let now = startedAt + const starts: number[] = [] + const waits: number[] = [] + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + waits.push(ms) + now += ms + }, + collect: async () => { + starts.push(now) + now += 15_000 + return healthySample(now) + }, + persist: async () => {}, + checkpoint: async () => {} + }) + expect(starts.slice(0, 3)).toEqual([ + startedAt, + startedAt + 60_000, + startedAt + 120_000 + ]) + expect(waits[0]).toBe(45_000) + }) +}) diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts new file mode 100644 index 00000000000..6073e351511 --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-monitor.ts @@ -0,0 +1,800 @@ +import { + exactAdmissionSelector, + type AdmissionSelector, + type AdmissionState +} from './incident-selector.js' + +export const INCIDENT_MONITOR_THRESHOLDS = { + activeProbeMaxAgeMs: 60_000, + cloudDataMaxAgeMs: 180_000, + relayLogMaxAgeMs: 180_000, + heartbeatMaxAgeMs: 45_000, + endpointLatencyMs: 2_000, + cloudSqlCpuUtilization: 0.8, + cloudSqlMemoryUtilization: 0.9, + // Why: healthy latest-sum backends idle near 100 but spike to 216 in 1-minute + // bursts (~10 min/day exceeded the old bar of 160 on 2026-08-26, freezing a + // pre-drain gate on baseline noise). 250 clears measured healthy peaks while + // firing well before the verified 400-connection ceiling; pool-wait and + // exhausted-retry signals keep their strict thresholds. + cloudSqlBackends: 250, + // Bound the observed recovery load; deadlocks remain zero-tolerance. + cloudSqlLockWaits: 20, + cloudSqlDeadlocks: 0, + // Why: pool amplitude cannot discriminate the 2026-08-23 incident. Healthy + // fleet-wide bursts reach 43 waiters / 2.03s waits several times an hour, + // and a cell roll's reconnect surge peaks at 676 waiters, while the real + // incident peaked at 356 waiters and never crossed 2.5s (waits cap ~2s + // structurally). The old bars of 30/1000 froze pre-drain gates on baseline + // noise (~17% per 15-minute window). Incident-class contention is caught by + // the retry signals below at ~10x separation; these bars now fence only + // genuinely unbounded queueing, which grows past both. + relayPoolWaiting: 800, + relayPoolWaitMs: 2_500, + // Why: successful lock retries are the contention machinery working, not harm. + // Healthy 2026-08-26 baseline bursts to 234/5min (26% of windows crossed the old + // bar of 20, set unmeasured at the monitor's 2026-07-28 birth); the 2026-08-23 + // incident ran ~2,200-3,000/5min. 300 clears healthy bursts with ~10x incident + // margin; relayPostgresRetryExhausted below stays at zero tolerance, so any + // transaction that terminally fails still freezes the gate. + relayPostgresRetries: 300, + relayPostgresRetryExhausted: 0, + // Why: public admission is a per-instance semaphore, so fleet assignment capacity is + // concurrency x instances. A floor of 1 let the 2026-08-04 collapse from five instances + // to two pass unnoticed, which is the exact failure this monitor exists to catch. Keep in + // step with relay_min_instances in infra/terraform/environments/production.tfvars. + directorInstancesMin: 5, + // Five serving instances plus one warm scale-to-zero rollback during recovery. + directorInstancesMax: 6, + directorCpuUtilization: 0.8, + directorMemoryUtilization: 0.8, + directorConcurrency: 64, + directorErrors: 0, + authErrors: 0, + // Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is + // the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve). + cellConnections: 500, + cellQueuedBytes: 48 * 1024 * 1024, + migrationBlocked: 0 +} as const + +export const INCIDENT_CHECKPOINT_MINUTES = [0, 5, 15, 30, 45, 60, 75, 90] as const +export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 25 * 60_000 + +export type IncidentSourceName = + | 'active-probe' + | 'cloud-monitoring' + | 'relay-logs' + | 'director-admin' + +export type IncidentMigrationPolicy = + | 'strict' + | 'recover-forward' + | 'capacity-transition' + +export type IncidentSignal = { + value: number + observedAt: string +} + +export type IncidentSource = { + observedAt: string + signals: Record +} + +export type IncidentCellExpectation = { + cellId: string + runtimeKnown: boolean + powered: boolean + expectedAdmissionState: AdmissionState +} + +export type IncidentSample = { + collectedAt: string + selector: AdmissionSelector + expectedSelector: AdmissionSelector + sources: Partial> + cells: IncidentCellExpectation[] +} + +export type IncidentFailure = { + code: string + source: IncidentSourceName + signal?: string + observed?: number + threshold?: number +} + +export type IncidentEvaluation = { + status: 'green' | 'freeze' + evaluatedAt: string + failures: IncidentFailure[] +} + +export type IncidentCheckpoint = { + schemaVersion: 4 + incidentId: string + environment: 'production' | 'staging' + expectedSelector: AdmissionSelector + preDrainDryRun: boolean + migrationPolicy: IncidentMigrationPolicy + recoverySourceCellId: string | null + capacityCellId: string | null + windowSequence: number + windowStartedAt: string + checkpointMinute: number + scheduledAt: string + recordedAt: string + status: 'green' | 'freeze' + frozenAt: string | null + sampleCount: number + failures: IncidentFailure[] + thresholds: typeof INCIDENT_MONITOR_THRESHOLDS +} + +export type IncidentMonitorState = { + schemaVersion: 4 + incidentId: string + environment: 'production' | 'staging' + expectedSelector: AdmissionSelector + preDrainDryRun: boolean + migrationPolicy: IncidentMigrationPolicy + recoverySourceCellId: string | null + capacityCellId: string | null + startedAt: string + windowStartedAt: string | null + windowSequence: number + durationMinutes: number + intervalMs: number + nextCheckpointIndex: number + sampleCount: number + totalSampleCount: number + lastSampleAt: string | null + continuityEvents: { + recordedAt: string + windowSequence: number + failures: IncidentFailure[] + }[] + frozenAt: string | null + failures: IncidentFailure[] + completedAt: string | null +} + +type NumericRule = { + source: IncidentSourceName + signal: string + comparison: 'max' | 'min' | 'equal' + threshold: number +} + +const NUMERIC_RULES: NumericRule[] = [ + { source: 'active-probe', signal: 'director.health', comparison: 'equal', threshold: 1 }, + { source: 'active-probe', signal: 'director.ready', comparison: 'equal', threshold: 1 }, + { + source: 'active-probe', + signal: 'director.latency_ms', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs + }, + { source: 'active-probe', signal: 'auth.health', comparison: 'equal', threshold: 1 }, + { + source: 'active-probe', + signal: 'auth.latency_ms', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs + }, + { + source: 'cloud-monitoring', + signal: 'cloud_sql.cpu', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlCpuUtilization + }, + { + source: 'cloud-monitoring', + signal: 'cloud_sql.memory', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlMemoryUtilization + }, + { + source: 'cloud-monitoring', + signal: 'cloud_sql.backends', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlBackends + }, + { + source: 'cloud-monitoring', + signal: 'director.instances', + comparison: 'min', + threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMin + }, + { + source: 'cloud-monitoring', + signal: 'director.instances', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMax + }, + { + source: 'cloud-monitoring', + signal: 'director.cpu', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.directorCpuUtilization + }, + { + source: 'cloud-monitoring', + signal: 'director.memory', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.directorMemoryUtilization + }, + { + source: 'cloud-monitoring', + signal: 'director.concurrency', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.directorConcurrency + }, + { + source: 'cloud-monitoring', + signal: 'director.errors', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.directorErrors + }, + { + source: 'cloud-monitoring', + signal: 'auth.errors', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.authErrors + }, + { + source: 'cloud-monitoring', + signal: 'cloud_sql.lock_waits', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlLockWaits + }, + { + source: 'cloud-monitoring', + signal: 'cloud_sql.deadlocks', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlDeadlocks + }, + { + source: 'relay-logs', + signal: 'relay.pool_waiting', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.relayPoolWaiting + }, + { + source: 'relay-logs', + signal: 'relay.pool_wait_ms', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.relayPoolWaitMs + }, + { + source: 'relay-logs', + signal: 'relay.postgres_retries', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries + }, + { + source: 'relay-logs', + signal: 'relay.postgres_retry_exhausted', + comparison: 'max', + threshold: INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetryExhausted + } +] + +const SOURCE_MAX_AGE: Record = { + 'active-probe': INCIDENT_MONITOR_THRESHOLDS.activeProbeMaxAgeMs, + 'cloud-monitoring': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs, + 'relay-logs': INCIDENT_MONITOR_THRESHOLDS.relayLogMaxAgeMs, + 'director-admin': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs +} + +function ageMs(timestamp: string, nowMs: number): number { + const parsed = Date.parse(timestamp) + return Number.isFinite(parsed) ? nowMs - parsed : Number.POSITIVE_INFINITY +} + +function addMissingSignal( + failures: IncidentFailure[], + source: IncidentSourceName, + signal: string +): void { + failures.push({ code: 'signal_missing', source, signal }) +} + +function checkRule( + failures: IncidentFailure[], + source: IncidentSourceName, + signals: Record, + rule: NumericRule +): void { + const signal = signals[rule.signal] + if (!signal) return addMissingSignal(failures, source, rule.signal) + const failed = + (rule.comparison === 'max' && signal.value > rule.threshold) || + (rule.comparison === 'min' && signal.value < rule.threshold) || + (rule.comparison === 'equal' && signal.value !== rule.threshold) + if (failed) { + failures.push({ + code: `threshold_${rule.comparison}`, + source, + signal: rule.signal, + observed: signal.value, + threshold: rule.threshold + }) + } +} + +function checkCell( + failures: IncidentFailure[], + sample: IncidentSample, + cell: IncidentCellExpectation, + migrationPolicy: IncidentMigrationPolicy, + recoverySourceCellId: string | null, + capacityCellId: string | null +): void { + const probe = sample.sources['active-probe']?.signals + const relay = sample.sources['relay-logs']?.signals + const admin = sample.sources['director-admin']?.signals + if (!cell.runtimeKnown) { + failures.push({ + code: 'runtime_power_unknown', + source: 'cloud-monitoring', + signal: `cell.${cell.cellId}.powered` + }) + } + if ( + cell.runtimeKnown && + cell.expectedAdmissionState !== 'existing-only' && + !cell.powered + ) { + failures.push({ + code: 'expected_admission_without_runtime', + source: 'director-admin', + signal: `cell.${cell.cellId}.powered`, + observed: 0, + threshold: 1 + }) + } + const checks = [ + ['active-probe', probe, `cell.${cell.cellId}.health`, cell.powered ? 1 : 0, 'equal'], + ['active-probe', probe, `cell.${cell.cellId}.ready`, cell.powered ? 1 : 0, 'equal'], + [ + 'active-probe', + probe, + `cell.${cell.cellId}.latency_ms`, + INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs, + 'max' + ], + [ + 'director-admin', + admin, + `cell.${cell.cellId}.admission_state`, + ['existing-only', 'migration-only', 'general'].indexOf( + cell.expectedAdmissionState + ), + 'equal' + ], + [ + 'director-admin', + admin, + `cell.${cell.cellId}.heartbeat_fresh`, + 1, + 'equal' + ], + [ + 'director-admin', + admin, + `cell.${cell.cellId}.heartbeat_age_ms`, + INCIDENT_MONITOR_THRESHOLDS.heartbeatMaxAgeMs, + 'max' + ], + [ + 'director-admin', + admin, + `cell.${cell.cellId}.migration_blocked`, + INCIDENT_MONITOR_THRESHOLDS.migrationBlocked, + 'max' + ], + [ + 'director-admin', + admin, + `cell.${cell.cellId}.migration_target_inactive`, + INCIDENT_MONITOR_THRESHOLDS.migrationBlocked, + 'max' + ], + [ + 'relay-logs', + relay, + `cell.${cell.cellId}.connections`, + (admin?.[`cell.${cell.cellId}.connection_hard_cap`]?.value ?? + INCIDENT_MONITOR_THRESHOLDS.cellConnections + 1) - 1, + 'max' + ], + [ + 'relay-logs', + relay, + `cell.${cell.cellId}.queued_bytes`, + INCIDENT_MONITOR_THRESHOLDS.cellQueuedBytes, + 'max' + ] + ] as const + for (const [source, signals, signalName, threshold, comparison] of checks) { + if ( + migrationPolicy === 'recover-forward' && + cell.cellId === recoverySourceCellId && + signalName.endsWith('.migration_target_inactive') + ) { + if (!signals?.[signalName]) addMissingSignal(failures, source, signalName) + continue + } + if ( + migrationPolicy === 'capacity-transition' && + capacityCellId !== null && + cell.cellId !== capacityCellId && + cell.expectedAdmissionState === 'existing-only' && + signalName.endsWith('.migration_target_inactive') + ) { + if (!signals?.[signalName]) addMissingSignal(failures, source, signalName) + continue + } + if ( + cell.expectedAdmissionState === 'existing-only' && + signalName.endsWith('.connections') + ) { + continue + } + if ( + !cell.powered && + [ + 'latency_ms', + 'heartbeat_fresh', + 'heartbeat_age_ms', + 'connections', + 'queued_bytes' + ].some((suffix) => signalName.endsWith(suffix)) + ) { + continue + } + if (!signals?.[signalName]) { + addMissingSignal(failures, source, signalName) + continue + } + const value = signals[signalName].value + const failed = comparison === 'equal' ? value !== threshold : value > threshold + if (failed) { + failures.push({ + code: `threshold_${comparison}`, + source, + signal: signalName, + observed: value, + threshold + }) + } + } +} + +export function evaluateIncidentSample( + sample: IncidentSample, + nowMs = Date.now(), + migrationPolicy: IncidentMigrationPolicy = 'strict', + recoverySourceCellId: string | null = null, + capacityCellId: string | null = null +): IncidentEvaluation { + const failures: IncidentFailure[] = [] + if (!exactAdmissionSelector(sample.selector, sample.expectedSelector)) { + failures.push({ + code: 'selector_mismatch', + source: 'director-admin', + signal: 'selector.generation', + observed: sample.selector.generation, + threshold: sample.expectedSelector.generation + }) + } + for (const [sourceName, maxAge] of Object.entries(SOURCE_MAX_AGE) as [ + IncidentSourceName, + number + ][]) { + const source = sample.sources[sourceName] + if (!source) { + failures.push({ code: 'source_missing', source: sourceName }) + continue + } + if (ageMs(source.observedAt, nowMs) < 0 || ageMs(source.observedAt, nowMs) > maxAge) { + failures.push({ + code: 'source_stale', + source: sourceName, + observed: ageMs(source.observedAt, nowMs), + threshold: maxAge + }) + } + for (const [signalName, signal] of Object.entries(source.signals)) { + if (ageMs(signal.observedAt, nowMs) < 0 || ageMs(signal.observedAt, nowMs) > maxAge) { + failures.push({ + code: 'signal_stale', + source: sourceName, + signal: signalName, + observed: ageMs(signal.observedAt, nowMs), + threshold: maxAge + }) + } + } + } + for (const rule of NUMERIC_RULES) { + const source = sample.sources[rule.source] + if (source) checkRule(failures, rule.source, source.signals, rule) + } + for (const cell of sample.cells) { + checkCell( + failures, + sample, + cell, + migrationPolicy, + recoverySourceCellId, + capacityCellId + ) + } + return { + status: failures.length === 0 ? 'green' : 'freeze', + evaluatedAt: new Date(nowMs).toISOString(), + failures + } +} + +export function initialIncidentMonitorState(input: { + incidentId: string + environment: 'production' | 'staging' + expectedSelector: AdmissionSelector + preDrainDryRun: boolean + migrationPolicy: IncidentMigrationPolicy + recoverySourceCellId: string | null + capacityCellId: string | null + startedAt: string + durationMinutes: number + intervalMs: number +}): IncidentMonitorState { + if (input.intervalMs < 1_000 || input.intervalMs > 60_000) { + throw new Error('incident monitor interval must be between 1 and 60 seconds') + } + if (input.durationMinutes < 15 || input.durationMinutes > 90) { + throw new Error('incident monitor duration must be between 15 and 90 minutes') + } + return { + schemaVersion: 4, + ...input, + windowStartedAt: input.startedAt, + windowSequence: 0, + nextCheckpointIndex: 0, + sampleCount: 0, + totalSampleCount: 0, + lastSampleAt: null, + continuityEvents: [], + frozenAt: null, + failures: [], + completedAt: null + } +} + +export type IncidentMonitorDependencies = { + now(): number + wait(ms: number): Promise + collect(): Promise + persist(state: IncidentMonitorState): Promise + checkpoint(summary: IncidentCheckpoint): Promise +} + +function checkpointMinutes(durationMinutes: number): number[] { + return INCIDENT_CHECKPOINT_MINUTES.filter((minute) => minute <= durationMinutes) +} + +const CONTINUITY_FAILURE_CODES = new Set([ + 'collector_failed', + 'monitor_gap', + 'signal_stale', + 'source_missing', + 'source_stale' +]) + +function resetContinuousWindow( + state: IncidentMonitorState, + recordedAt: string, + failures: IncidentFailure[] +): void { + if (state.windowStartedAt !== null) { + state.windowSequence++ + state.windowStartedAt = null + state.nextCheckpointIndex = 0 + state.sampleCount = 0 + state.completedAt = null + } + state.continuityEvents.push({ + recordedAt, + windowSequence: state.windowSequence, + failures + }) +} + +function completeContinuityDeadline( + state: IncidentMonitorState, + nowMs: number, + lineageStartMs: number +): void { + const recordedAt = new Date(nowMs).toISOString() + state.frozenAt ??= recordedAt + state.failures.push({ + code: 'continuity_deadline_exceeded', + source: 'active-probe', + observed: nowMs - lineageStartMs, + threshold: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + }) + state.completedAt = recordedAt +} + +export async function runIncidentMonitor( + initialState: IncidentMonitorState, + dependencies: IncidentMonitorDependencies +): Promise { + const state = structuredClone(initialState) + const lineageStartMs = Date.parse(state.startedAt) + if (!Number.isFinite(lineageStartMs)) { + throw new Error('incident monitor start time is invalid') + } + const checkpoints = checkpointMinutes(state.durationMinutes) + const lineageDeadlineMs = state.preDrainDryRun + ? lineageStartMs + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + : Number.POSITIVE_INFINITY + const resumedAt = dependencies.now() + const priorSampleMs = state.lastSampleAt + ? Date.parse(state.lastSampleAt) + : lineageStartMs + const gapThreshold = state.intervalMs + if (resumedAt - priorSampleMs > gapThreshold) { + resetContinuousWindow(state, new Date(resumedAt).toISOString(), [{ + code: 'monitor_gap', + source: 'active-probe', + observed: resumedAt - priorSampleMs, + threshold: gapThreshold + }]) + } + if (state.completedAt !== null) { + await dependencies.persist(state) + return state + } + while (state.completedAt === null) { + if (dependencies.now() > lineageDeadlineMs) { + completeContinuityDeadline(state, dependencies.now(), lineageStartMs) + await dependencies.persist(state) + break + } + const sampleStartedAt = dependencies.now() + let evaluation: IncidentEvaluation + try { + evaluation = evaluateIncidentSample( + await dependencies.collect(), + dependencies.now(), + state.migrationPolicy, + state.recoverySourceCellId, + state.capacityCellId + ) + } catch { + evaluation = { + status: 'freeze', + evaluatedAt: new Date(dependencies.now()).toISOString(), + failures: [{ + code: 'collector_failed', + source: 'cloud-monitoring' + }] + } + } + state.totalSampleCount++ + state.lastSampleAt = evaluation.evaluatedAt + const continuityFailures = evaluation.failures.filter((failure) => + CONTINUITY_FAILURE_CODES.has(failure.code) + ) + const thresholdFailures = evaluation.failures.filter((failure) => + !CONTINUITY_FAILURE_CODES.has(failure.code) + ) + if (continuityFailures.length > 0) { + resetContinuousWindow(state, evaluation.evaluatedAt, continuityFailures) + } else { + if (state.windowStartedAt === null) { + state.windowStartedAt = evaluation.evaluatedAt + } + state.sampleCount++ + } + if (thresholdFailures.length > 0) { + state.frozenAt ??= evaluation.evaluatedAt + state.failures = [...state.failures, ...thresholdFailures] + } + if (state.windowStartedAt === null) { + if (dependencies.now() >= lineageDeadlineMs) { + completeContinuityDeadline(state, dependencies.now(), lineageStartMs) + await dependencies.persist(state) + break + } + await dependencies.persist(state) + await dependencies.wait( + Math.max(0, Math.min(state.intervalMs, lineageDeadlineMs - dependencies.now())) + ) + continue + } + const startMs = Date.parse(state.windowStartedAt) + const endMs = startMs + state.durationMinutes * 60_000 + const elapsedMinutes = (dependencies.now() - startMs) / 60_000 + while ( + state.nextCheckpointIndex < checkpoints.length && + elapsedMinutes >= checkpoints[state.nextCheckpointIndex]! + ) { + const minute = checkpoints[state.nextCheckpointIndex]! + await dependencies.checkpoint({ + schemaVersion: 4, + incidentId: state.incidentId, + environment: state.environment, + expectedSelector: state.expectedSelector, + preDrainDryRun: state.preDrainDryRun, + migrationPolicy: state.migrationPolicy, + recoverySourceCellId: state.recoverySourceCellId, + capacityCellId: state.capacityCellId, + windowSequence: state.windowSequence, + windowStartedAt: state.windowStartedAt, + checkpointMinute: minute, + scheduledAt: new Date(startMs + minute * 60_000).toISOString(), + recordedAt: new Date(dependencies.now()).toISOString(), + status: state.frozenAt ? 'freeze' : 'green', + frozenAt: state.frozenAt, + sampleCount: state.sampleCount, + failures: state.failures, + thresholds: INCIDENT_MONITOR_THRESHOLDS + }) + state.nextCheckpointIndex++ + } + if (state.preDrainDryRun && state.frozenAt !== null) { + state.completedAt = new Date(dependencies.now()).toISOString() + await dependencies.persist(state) + break + } + if (dependencies.now() >= endMs) { + state.completedAt = new Date(dependencies.now()).toISOString() + await dependencies.persist(state) + break + } + if (dependencies.now() >= lineageDeadlineMs) { + completeContinuityDeadline(state, dependencies.now(), lineageStartMs) + await dependencies.persist(state) + break + } + await dependencies.persist(state) + await dependencies.wait( + Math.max( + 0, + Math.min(sampleStartedAt + state.intervalMs, endMs, lineageDeadlineMs) - dependencies.now() + ) + ) + } + return state +} + +export function preDrainDryRunPassed(state: Pick< + IncidentMonitorState, + | 'completedAt' + | 'durationMinutes' + | 'frozenAt' + | 'intervalMs' + | 'preDrainDryRun' + | 'sampleCount' + | 'startedAt' +>): boolean { + const minimumSamples = + Math.ceil((state.durationMinutes * 60_000) / state.intervalMs) + 1 + const lineageElapsedMs = state.completedAt === null + ? Number.POSITIVE_INFINITY + : Date.parse(state.completedAt) - Date.parse(state.startedAt) + return ( + state.preDrainDryRun && + state.durationMinutes === 15 && + state.completedAt !== null && + lineageElapsedMs >= 0 && + lineageElapsedMs <= INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS && + state.frozenAt === null && + state.sampleCount >= minimumSamples + ) +} diff --git a/cloud/apps/relay-ops/src/incident-selector.ts b/cloud/apps/relay-ops/src/incident-selector.ts new file mode 100644 index 00000000000..7e7ddd91e59 --- /dev/null +++ b/cloud/apps/relay-ops/src/incident-selector.ts @@ -0,0 +1,77 @@ +import { z } from 'zod' + +export const AdmissionStateSchema = z.enum([ + 'existing-only', + 'migration-only', + 'general' +]) + +export type AdmissionState = z.infer + +export const SelectorMembershipSchema = z.object({ + existingOnly: z.array(z.string()), + migrationOnly: z.array(z.string()), + general: z.array(z.string()) +}) + +export type SelectorMembership = z.infer + +export const AdmissionSelectorSchema = z.object({ + generation: z.number().int().nonnegative(), + membership: SelectorMembershipSchema +}) + +export type AdmissionSelector = z.infer + +export function normalizeSelectorMembership( + membership: SelectorMembership, + configuredCellIds: ReadonlySet +): SelectorMembership { + const normalized = { + existingOnly: [...membership.existingOnly].sort(), + migrationOnly: [...membership.migrationOnly].sort(), + general: [...membership.general].sort() + } + const all = [ + ...normalized.existingOnly, + ...normalized.migrationOnly, + ...normalized.general + ] + if ( + all.length !== configuredCellIds.size || + new Set(all).size !== all.length || + all.some((cellId) => !configuredCellIds.has(cellId)) + ) { + throw new Error('selector membership must contain every configured cell exactly once') + } + return normalized +} + +export function selectorCellState( + selector: AdmissionSelector, + cellId: string +): AdmissionState { + if (selector.membership.existingOnly.includes(cellId)) return 'existing-only' + if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only' + if (selector.membership.general.includes(cellId)) return 'general' + throw new Error(`selector does not contain ${cellId}`) +} + +export function effectiveAdmissionState( + selector: AdmissionSelector, + legacyEnabled: boolean, + cellId: string +): AdmissionState { + if (selector.generation === 0) return legacyEnabled ? 'general' : 'existing-only' + return selectorCellState(selector, cellId) +} + +export function exactAdmissionSelector( + actual: AdmissionSelector, + expected: AdmissionSelector +): boolean { + return ( + actual.generation === expected.generation && + JSON.stringify(actual.membership) === JSON.stringify(expected.membership) + ) +} diff --git a/cloud/apps/relay-ops/src/index.ts b/cloud/apps/relay-ops/src/index.ts new file mode 100644 index 00000000000..0544cfe7595 --- /dev/null +++ b/cloud/apps/relay-ops/src/index.ts @@ -0,0 +1,108 @@ +import { randomBytes, timingSafeEqual } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { resolve } from 'node:path' +import { serve } from '@hono/node-server' +import { Hono } from 'hono' +import { z } from 'zod' +import { DashboardSnapshotCache } from './dashboard-snapshot.js' +import { createGcloudClient } from './gcloud-client.js' +import { + dispatchStagingPowerWorkflow, + parseStagingPowerRequest +} from './staging-workflow.js' + +const QuerySchema = z.object({ + environment: z.enum(['production', 'staging']).default('production'), + window: z.coerce.number().int().min(30).max(1440).default(360) +}) +const port = z.coerce.number().int().min(1024).max(65_535).parse(process.env.PORT ?? 2455) +const controlsEnabled = process.env.RELAY_OPS_ENABLE_STAGING_CONTROLS === '1' +const csrfToken = randomBytes(32).toString('base64url') +const publicDirectory = resolve(import.meta.dirname, '../public') +const cache = new DashboardSnapshotCache(createGcloudClient()) +const app = new Hono() + +function safeEqual(left: string, right: string): boolean { + const leftBuffer = Buffer.from(left) + const rightBuffer = Buffer.from(right) + return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer) +} + +app.use('*', async (context, next) => { + await next() + context.header('Cache-Control', 'no-store') + context.header('Content-Security-Policy', [ + "default-src 'self'", + "script-src 'self'", + "style-src 'self'", + "font-src 'self'", + "connect-src 'self'", + "img-src 'self' data:", + "object-src 'none'", + "base-uri 'none'", + "frame-ancestors 'none'", + "form-action 'self'" + ].join('; ')) + context.header('Referrer-Policy', 'no-referrer') + context.header('X-Content-Type-Options', 'nosniff') + context.header('X-Frame-Options', 'DENY') +}) + +app.get('/health', (context) => context.json({ status: 'ok' })) +app.get('/api/config', (context) => context.json({ + stagingControlsEnabled: controlsEnabled, + csrfToken: controlsEnabled ? csrfToken : null +})) +app.get('/api/snapshot', async (context) => { + const query = QuerySchema.safeParse(context.req.query()) + if (!query.success) return context.json({ error: 'Invalid dashboard query' }, 400) + try { + return context.json(await cache.read(query.data.environment, query.data.window)) + } catch { + return context.json({ + error: 'Relay operations data is unavailable. Check local gcloud and gh authentication.' + }, 503) + } +}) +app.post('/api/staging/power', async (context) => { + if (!controlsEnabled) return context.json({ error: 'Staging controls are disabled' }, 403) + const origin = context.req.header('origin') + const expectedOrigin = `http://127.0.0.1:${port}` + if (origin !== expectedOrigin) return context.json({ error: 'Origin rejected' }, 403) + if (!safeEqual(context.req.header('x-csrf-token') ?? '', csrfToken)) { + return context.json({ error: 'Request token rejected' }, 403) + } + try { + const request = parseStagingPowerRequest(await context.req.json()) + await dispatchStagingPowerWorkflow(request) + return context.json({ accepted: true }) + } catch { + return context.json({ error: 'Invalid or failed staging workflow dispatch' }, 400) + } +}) + +const staticTypes: Record = { + '/app.js': 'text/javascript; charset=utf-8', + '/styles.css': 'text/css; charset=utf-8' +} +for (const [route, contentType] of Object.entries(staticTypes)) { + app.get(route, async (context) => { + const file = route.slice(1) + try { + const content = await readFile(resolve(publicDirectory, file)) + return context.body(content, 200, { 'Content-Type': contentType }) + } catch { + return context.notFound() + } + }) +} +app.get('/', async (context) => { + const html = await readFile(resolve(publicDirectory, 'index.html'), 'utf8') + return context.html(html) +}) + +serve({ fetch: app.fetch, hostname: '127.0.0.1', port }, () => { + // Loopback is intentional; operators may add authenticated Tailscale Serve separately. + console.log(`Orca Relay Operations: http://127.0.0.1:${port}`) + console.log(`Staging controls: ${controlsEnabled ? 'enabled through GitHub workflow' : 'read-only'}`) +}) diff --git a/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts b/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts new file mode 100644 index 00000000000..3bcd40c3c4b --- /dev/null +++ b/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' +import { RELAY_METRICS, readMonitoringSnapshot } from './monitoring-snapshot.js' +import type { GcloudClient } from './gcloud-client.js' +import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' + +const gcloud: GcloudClient = { + accessToken: async () => 'a'.repeat(40) +} + +function distribution(at: string, mean: number | undefined, count = 1) { + return { + interval: { endTime: at }, + value: { distributionValue: { count, ...(mean === undefined ? {} : { mean }) } } + } +} + +describe('readMonitoringSnapshot', () => { + it('aggregates gauge series per minute and distribution deltas by count', async () => { + const fetchImpl: typeof fetch = async (input) => { + const url = new URL(String(input)) + if (url.pathname.endsWith('/alertPolicies')) { + return Response.json({ alertPolicies: [] }) + } + const filter = url.searchParams.get('filter') ?? '' + if (filter.includes('orca_relay_controls')) { + return Response.json({ timeSeries: [ + { + metric: { labels: { cell_id: 'production-gce-c1' } }, + resource: { type: 'gce_instance', labels: { instance_id: 'one' } }, + points: [ + distribution('2026-07-15T12:00:10Z', 1), + distribution('2026-07-15T12:00:50Z', 2) + ] + }, + { + metric: { labels: { cell_id: 'production-gce-c1' } }, + resource: { type: 'gce_instance', labels: { instance_id: 'two' } }, + points: [distribution('2026-07-15T12:00:20Z', 3)] + }, + { + metric: { labels: { cell_id: 'production-gce-c1' } }, + resource: { type: 'gce_instance', labels: { instance_id: 'stale' } }, + points: [distribution('2026-07-15T11:59:20Z', 100)] + } + ] }) + } + if (filter.includes('orca_relay_forwarded_bytes')) { + return Response.json({ timeSeries: [{ + metric: { labels: { cell_id: 'production-gce-c1' } }, + resource: { type: 'gce_instance', labels: { instance_id: 'one' } }, + points: [distribution('2026-07-15T12:00:20Z', 10, 4)] + }] }) + } + return Response.json({ timeSeries: [] }) + } + + const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, gcloud, { + now: new Date('2026-07-15T12:01:00Z'), + windowMinutes: 30, + fetchImpl + }) + + expect(result.warnings).toEqual([]) + expect(result.metrics.controls.points).toEqual([ + { at: '2026-07-15T11:59:00.000Z', value: 100 }, + { at: '2026-07-15T12:00:00.000Z', value: 5 } + ]) + expect(result.metrics.controls.latestByCell).toEqual({ 'production-gce-c1': 5 }) + expect(result.metrics.forwarded_bytes.latest).toBe(40) + expect(result.metrics.postgres_retries.available).toBe(true) + expect(Object.keys(result.metrics)).toHaveLength(RELAY_METRICS.length) + }) + + it('degrades safely when credentials are unavailable', async () => { + const unavailable: GcloudClient = { + accessToken: async () => { throw new Error('sensitive context') } + } + const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, unavailable) + expect(result.warnings).toEqual([ + 'Cloud Monitoring credentials are unavailable. Run gcloud auth login.' + ]) + expect(result.metrics.postgres_retries.available).toBe(false) + expect(JSON.stringify(result)).not.toContain('sensitive context') + }) +}) diff --git a/cloud/apps/relay-ops/src/monitoring-snapshot.ts b/cloud/apps/relay-ops/src/monitoring-snapshot.ts new file mode 100644 index 00000000000..9b5fe44bd4a --- /dev/null +++ b/cloud/apps/relay-ops/src/monitoring-snapshot.ts @@ -0,0 +1,379 @@ +import { z } from 'zod' +import type { RelayOpsEnvironment } from './environment-config.js' +import type { GcloudClient } from './gcloud-client.js' + +type MetricMode = 'gauge-sum' | 'delta-sum' | 'maximum' + +export type RelayMetricName = + | 'total_connections' + | 'controls' + | 'splices' + | 'pending_splices' + | 'queued_bytes' + | 'http_latency_ms' + | 'sql_latency_ms' + | 'heap_used_bytes' + | 'event_loop_ms_p99' + | 'forwarded_bytes' + | 'auth_successes' + | 'auth_failures' + | 'reconnects' + | 'sql_queries' + | 'sql_failures' + | 'assignment_5xx' + | 'postgres_retries' + | 'postgres_retry_exhausted' + | 'db_pool_total' + | 'db_pool_idle' + | 'db_pool_waiting' + | 'db_waiters_max' + | 'db_oldest_wait_ms' + | 'db_wait_ms_max' + +type MetricDefinition = { + name: RelayMetricName + label: string + unit: 'count' | 'bytes' | 'milliseconds' + mode: MetricMode +} + +export const RELAY_METRICS: MetricDefinition[] = [ + { name: 'total_connections', label: 'Connections', unit: 'count', mode: 'gauge-sum' }, + { name: 'controls', label: 'Desktop controls', unit: 'count', mode: 'gauge-sum' }, + { name: 'splices', label: 'Phone splices', unit: 'count', mode: 'gauge-sum' }, + { name: 'pending_splices', label: 'Pending splices', unit: 'count', mode: 'gauge-sum' }, + { name: 'queued_bytes', label: 'Queued bytes', unit: 'bytes', mode: 'maximum' }, + { name: 'http_latency_ms', label: 'HTTP latency', unit: 'milliseconds', mode: 'maximum' }, + { name: 'sql_latency_ms', label: 'SQL latency', unit: 'milliseconds', mode: 'maximum' }, + { name: 'heap_used_bytes', label: 'Heap used', unit: 'bytes', mode: 'maximum' }, + { + name: 'event_loop_ms_p99', + label: 'Event-loop p99', + unit: 'milliseconds', + mode: 'maximum' + }, + { name: 'forwarded_bytes', label: 'Forwarded bytes', unit: 'bytes', mode: 'delta-sum' }, + { name: 'auth_successes', label: 'Auth successes', unit: 'count', mode: 'delta-sum' }, + { name: 'auth_failures', label: 'Auth failures', unit: 'count', mode: 'delta-sum' }, + { name: 'reconnects', label: 'Reconnects', unit: 'count', mode: 'delta-sum' }, + { name: 'sql_queries', label: 'SQL queries', unit: 'count', mode: 'delta-sum' }, + { name: 'sql_failures', label: 'SQL failures', unit: 'count', mode: 'delta-sum' }, + { name: 'assignment_5xx', label: 'Assignment 5xx', unit: 'count', mode: 'delta-sum' }, + { + name: 'postgres_retries', + label: 'PostgreSQL retries', + unit: 'count', + mode: 'delta-sum' + }, + { + name: 'postgres_retry_exhausted', + label: 'PostgreSQL retry exhausted', + unit: 'count', + mode: 'delta-sum' + }, + { name: 'db_pool_total', label: 'Database pool total', unit: 'count', mode: 'gauge-sum' }, + { name: 'db_pool_idle', label: 'Database pool idle', unit: 'count', mode: 'gauge-sum' }, + { + name: 'db_pool_waiting', + label: 'Database pool waiting', + unit: 'count', + mode: 'gauge-sum' + }, + { + name: 'db_waiters_max', + label: 'Database waiters max', + unit: 'count', + mode: 'maximum' + }, + { + name: 'db_oldest_wait_ms', + label: 'Database oldest wait', + unit: 'milliseconds', + mode: 'maximum' + }, + { + name: 'db_wait_ms_max', + label: 'Database wait max', + unit: 'milliseconds', + mode: 'maximum' + } +] + +const NumericSchema = z.union([z.number(), z.string()]).transform((value) => Number(value)) +const DistributionSchema = z.object({ + count: NumericSchema.default(0), + mean: NumericSchema.optional() +}) +const PointSchema = z.object({ + interval: z.object({ endTime: z.string() }), + value: z.object({ + doubleValue: NumericSchema.optional(), + int64Value: NumericSchema.optional(), + distributionValue: DistributionSchema.optional() + }) +}) +const TimeSeriesSchema = z.object({ + metric: z.object({ labels: z.record(z.string()).default({}) }), + resource: z.object({ type: z.string(), labels: z.record(z.string()).default({}) }), + points: z.array(PointSchema).default([]) +}) +const TimeSeriesResponseSchema = z.object({ + timeSeries: z.array(TimeSeriesSchema).default([]) +}) + +export type MetricPoint = { at: string; value: number } + +export type RelayMetricSnapshot = MetricDefinition & { + available: boolean + points: MetricPoint[] + latest: number | null + latestAt: string | null + latestByCell: Record +} + +export type AlertPolicySnapshot = { + id: string + displayName: string + enabled: boolean + documentation: string | null +} + +export type MonitoringSnapshot = { + startAt: string + endAt: string + resolutionSeconds: number + metrics: Record + alertPolicies: AlertPolicySnapshot[] + warnings: string[] +} + +type ParsedPoint = { + atMs: number + bucketMs: number + value: number + sampleTotal: number + seriesKey: string + cellId: string +} + +function pointValue(point: z.infer): { value: number; sampleTotal: number } { + if (point.value.distributionValue) { + const count = point.value.distributionValue.count + const value = point.value.distributionValue.mean ?? 0 + return { value, sampleTotal: value * count } + } + const value = point.value.doubleValue ?? point.value.int64Value ?? 0 + return { value, sampleTotal: value } +} + +function parsePoints(series: z.infer[]): ParsedPoint[] { + return series.flatMap((entry) => { + const cellId = entry.metric.labels.cell_id ?? 'unknown' + const resourceId = + entry.resource.labels.instance_id ?? entry.resource.labels.revision_name ?? entry.resource.type + const seriesKey = `${cellId}:${resourceId}` + return entry.points.flatMap((point) => { + const atMs = Date.parse(point.interval.endTime) + if (!Number.isFinite(atMs)) return [] + const values = pointValue(point) + return [{ + atMs, + bucketMs: Math.floor(atMs / 60_000) * 60_000, + value: values.value, + sampleTotal: values.sampleTotal, + seriesKey, + cellId + }] + }) + }) +} + +function aggregatePoints(points: ParsedPoint[], mode: MetricMode): MetricPoint[] { + if (mode === 'gauge-sum') { + const buckets = new Map>() + for (const point of points) { + const bySeries = buckets.get(point.bucketMs) ?? new Map() + const previous = bySeries.get(point.seriesKey) + if (!previous || previous.atMs < point.atMs) bySeries.set(point.seriesKey, point) + buckets.set(point.bucketMs, bySeries) + } + return [...buckets.entries()] + .sort(([left], [right]) => left - right) + .map(([at, bySeries]) => ({ + at: new Date(at).toISOString(), + value: [...bySeries.values()].reduce((total, point) => total + point.value, 0) + })) + } + const buckets = new Map() + for (const point of points) { + const value = mode === 'delta-sum' ? point.sampleTotal : point.value + const previous = buckets.get(point.bucketMs) + buckets.set( + point.bucketMs, + mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value + ) + } + return [...buckets.entries()] + .sort(([left], [right]) => left - right) + .map(([at, value]) => ({ at: new Date(at).toISOString(), value })) +} + +function latestByCell(points: ParsedPoint[], mode: MetricMode): Record { + const newestBucketByCell = new Map() + for (const point of points) { + newestBucketByCell.set( + point.cellId, + Math.max(newestBucketByCell.get(point.cellId) ?? 0, point.bucketMs) + ) + } + const newestBySeries = new Map() + for (const point of points) { + if (point.bucketMs !== newestBucketByCell.get(point.cellId)) continue + const previous = newestBySeries.get(point.seriesKey) + if (!previous || previous.atMs < point.atMs) newestBySeries.set(point.seriesKey, point) + } + const totals = new Map() + for (const point of newestBySeries.values()) { + const value = mode === 'delta-sum' ? point.sampleTotal : point.value + const previous = totals.get(point.cellId) + totals.set( + point.cellId, + mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value + ) + } + return Object.fromEntries(totals) +} + +async function monitoringRequest( + fetchImpl: typeof fetch, + token: string, + url: URL +): Promise { + const response = await fetchImpl(url, { + headers: { authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(30_000) + }) + if (!response.ok) throw new Error(`Cloud Monitoring returned ${response.status}`) + return await response.json() +} + +async function readMetric( + environment: RelayOpsEnvironment, + definition: MetricDefinition, + token: string, + startAt: string, + endAt: string, + fetchImpl: typeof fetch +): Promise { + const url = new URL( + `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries` + ) + url.searchParams.set( + 'filter', + `metric.type="logging.googleapis.com/user/orca_relay_${definition.name}"` + ) + url.searchParams.set('interval.startTime', startAt) + url.searchParams.set('interval.endTime', endAt) + url.searchParams.set('view', 'FULL') + url.searchParams.set('pageSize', '1000') + const body = TimeSeriesResponseSchema.parse( + await monitoringRequest(fetchImpl, token, url) + ) + const parsed = parsePoints(body.timeSeries) + const points = aggregatePoints(parsed, definition.mode) + const latest = points.at(-1) ?? null + return { + ...definition, + available: true, + points, + latest: latest?.value ?? null, + latestAt: latest?.at ?? null, + latestByCell: latestByCell(parsed, definition.mode) + } +} + +async function readAlertPolicies( + environment: RelayOpsEnvironment, + token: string, + fetchImpl: typeof fetch +): Promise { + const url = new URL( + `https://monitoring.googleapis.com/v3/projects/${environment.project}/alertPolicies` + ) + url.searchParams.set('pageSize', '100') + const body = (await monitoringRequest(fetchImpl, token, url)) as { + alertPolicies?: Array<{ + name?: string + displayName?: string + enabled?: boolean + documentation?: { content?: string } + }> + } + return (body.alertPolicies ?? []) + .filter((policy) => policy.displayName?.startsWith('Orca Relay:')) + .map((policy) => ({ + id: policy.name ?? '', + displayName: policy.displayName ?? 'Orca Relay alert', + enabled: policy.enabled === true, + documentation: policy.documentation?.content ?? null + })) + .sort((left, right) => left.displayName.localeCompare(right.displayName)) +} + +function emptyMetric(definition: MetricDefinition): RelayMetricSnapshot { + return { + ...definition, + available: false, + points: [], + latest: null, + latestAt: null, + latestByCell: {} + } +} + +export async function readMonitoringSnapshot( + environment: RelayOpsEnvironment, + gcloud: GcloudClient, + options: { now?: Date; windowMinutes?: number; fetchImpl?: typeof fetch } = {} +): Promise { + const now = options.now ?? new Date() + const windowMinutes = Math.min(24 * 60, Math.max(30, options.windowMinutes ?? 360)) + const endAt = now.toISOString() + const startAt = new Date(now.getTime() - windowMinutes * 60_000).toISOString() + const fetchImpl = options.fetchImpl ?? fetch + const warnings: string[] = [] + let token: string + try { + token = await gcloud.accessToken() + } catch { + return { + startAt, + endAt, + resolutionSeconds: 60, + metrics: Object.fromEntries( + RELAY_METRICS.map((definition) => [definition.name, emptyMetric(definition)]) + ) as Record, + alertPolicies: [], + warnings: ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.'] + } + } + const settled = await Promise.allSettled( + RELAY_METRICS.map((definition) => + readMetric(environment, definition, token, startAt, endAt, fetchImpl) + ) + ) + const metrics = {} as Record + settled.forEach((result, index) => { + const definition = RELAY_METRICS[index]! + if (result.status === 'fulfilled') metrics[definition.name] = result.value + else { + metrics[definition.name] = emptyMetric(definition) + warnings.push(`${definition.label} metric is unavailable.`) + } + }) + const alertPolicies = await readAlertPolicies(environment, token, fetchImpl).catch(() => { + warnings.push('Cloud Monitoring alert policies are unavailable.') + return [] + }) + return { startAt, endAt, resolutionSeconds: 60, metrics, alertPolicies, warnings } +} diff --git a/cloud/apps/relay-ops/src/relay-repository.test.ts b/cloud/apps/relay-ops/src/relay-repository.test.ts new file mode 100644 index 00000000000..e6a87c35e82 --- /dev/null +++ b/cloud/apps/relay-ops/src/relay-repository.test.ts @@ -0,0 +1,42 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { + RELAY_GITHUB_REPOSITORY, + RELAY_WORKFLOW_FILE_PREFIX, + relayRepositoryApiPath, + relayWorkflowFile +} from './relay-repository.js' + +const sourceDir = fileURLToPath(new URL('.', import.meta.url)) +const sources = readdirSync(sourceDir) + .filter((name) => name.endsWith('.ts') && name !== 'relay-repository.ts') + .map((name) => ({ name, text: readFileSync(`${sourceDir}${name}`, 'utf8') })) + +describe('relay repository identity', () => { + it('builds API paths and workflow filenames from the one repository name', () => { + expect(relayRepositoryApiPath('actions/runs')).toBe(`repos/${RELAY_GITHUB_REPOSITORY}/actions/runs`) + expect(relayWorkflowFile('power-relay-staging.yml')).toBe( + `${RELAY_WORKFLOW_FILE_PREFIX}power-relay-staging.yml` + ) + }) + + // Why: the public-repo copy renames the repository and prefixes every workflow file. Both have to + // be one edit, so no other module may restate either. + it('is the only module naming a GitHub repository', () => { + for (const { name, text } of sources) { + expect(text, `${name} restates a GitHub repository`).not.toMatch(/stablyai\//) + } + }) + + it('is the only module naming a workflow file', () => { + for (const { name, text } of sources) { + for (const match of text.matchAll(/'([^']*\.yml)'/g)) { + const file = match[1] ?? '' + expect(text, `${name} names ${file} outside relayWorkflowFile`).toMatch( + new RegExp(`relayWorkflowFile\\('${file.replaceAll('.', '\\.')}'\\)`) + ) + } + } + }) +}) diff --git a/cloud/apps/relay-ops/src/relay-repository.ts b/cloud/apps/relay-ops/src/relay-repository.ts new file mode 100644 index 00000000000..85158670adb --- /dev/null +++ b/cloud/apps/relay-ops/src/relay-repository.ts @@ -0,0 +1,14 @@ +// Single place naming the GitHub repository that holds the Relay workflows. When the Relay tree is +// copied to its public repository, only this file changes: the repository moves and every workflow +// file gains a prefix, while the workflow display names stay as they are. +export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca-cloud' + +export const RELAY_WORKFLOW_FILE_PREFIX = '' + +export function relayWorkflowFile(name: string): string { + return `${RELAY_WORKFLOW_FILE_PREFIX}${name}` +} + +export function relayRepositoryApiPath(resource: string): string { + return `repos/${RELAY_GITHUB_REPOSITORY}/${resource}` +} diff --git a/cloud/apps/relay-ops/src/resource-inventory.test.ts b/cloud/apps/relay-ops/src/resource-inventory.test.ts new file mode 100644 index 00000000000..6d8b3070c98 --- /dev/null +++ b/cloud/apps/relay-ops/src/resource-inventory.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from 'vitest' +import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' +import type { GcloudClient } from './gcloud-client.js' +import { probeEndpointHealth, readResourceInventory } from './resource-inventory.js' + +const digest = `sha256:${'a'.repeat(64)}` +const runService = { + template: { + scaling: { minInstanceCount: 0, maxInstanceCount: 2 }, + containers: [{ image: 'registry/image:tag' }] + }, + conditions: [{ state: 'CONDITION_SUCCEEDED' }], + latestReadyRevision: 'projects/project/revisions/revision-one' +} + +describe('readResourceInventory', () => { + it('does not delay a healthy endpoint sample', async () => { + let calls = 0 + let waits = 0 + const result = await probeEndpointHealth( + 'https://c9.relay.onorca.dev', + async () => { + calls += 1 + return new Response(null, { status: 200 }) + }, + async () => { + waits += 1 + } + ) + + expect(result.health).toBe(true) + expect(result.ready).toBe(true) + expect(calls).toBe(2) + expect(waits).toBe(0) + }) + + it('retries one transient endpoint failure within the same sample', async () => { + const calls = new Map() + const waits: number[] = [] + const result = await probeEndpointHealth( + 'https://c9.relay.onorca.dev', + async (input) => { + const path = new URL(String(input)).pathname + const call = (calls.get(path) ?? 0) + 1 + calls.set(path, call) + return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 }) + }, + async (ms) => { + waits.push(ms) + } + ) + + expect(result.health).toBe(true) + expect(result.ready).toBe(true) + expect(calls).toEqual(new Map([['/health', 2], ['/ready', 2]])) + expect(waits).toEqual([11_000]) + }) + + it('fails closed when the endpoint retry is also unhealthy', async () => { + let calls = 0 + const waits: number[] = [] + const result = await probeEndpointHealth( + 'https://c9.relay.onorca.dev', + async () => { + calls += 1 + return new Response(null, { status: 503 }) + }, + async (ms) => { + waits.push(ms) + } + ) + + expect(result.health).toBe(false) + expect(result.ready).toBe(false) + expect(calls).toBe(4) + expect(waits).toEqual([11_000]) + }) + + it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => { + const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) } + let publicProbeCalls = 0 + const fetchImpl: typeof fetch = async (input) => { + const url = new URL(String(input)) + if (url.hostname.endsWith('onorca.dev')) { + publicProbeCalls += 1 + return Response.json({ status: 'ok' }) + } + if (url.hostname === 'run.googleapis.com') return Response.json(runService) + if (url.hostname === 'sqladmin.googleapis.com') return Response.json({ + state: 'STOPPED', + databaseVersion: 'POSTGRES_17', + settings: { + activationPolicy: 'NEVER', + availabilityType: 'ZONAL', + tier: 'db-custom-1-3840' + } + }) + if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({ + managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' } + }) + if (url.pathname.includes('/instanceGroupManagers/')) { + const name = url.pathname.split('/').at(-1)! + return Response.json({ + name, + targetSize: 0, + size: '0', + instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`, + instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`, + status: { isStable: true } + }) + } + if (url.pathname.includes('/instanceTemplates/')) return Response.json({ + properties: { metadata: { items: [{ + key: 'startup-script', + value: `SECRET_TEXT\nORCA_RELAY_IMAGE_DIGEST=%s\\n' '${digest}'` + }] } } + }) + if (url.pathname.endsWith('/getHealth')) return Response.json([]) + throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`) + } + + const result = await readResourceInventory( + RELAY_OPS_ENVIRONMENTS.staging, + gcloud, + fetchImpl + ) + + expect(publicProbeCalls).toBe(0) + expect(result.cells.every((cell) => cell.targetSize === 0)).toBe(true) + expect(result.cells.every((cell) => cell.endpoint.health === null)).toBe(true) + expect(result.cells.every((cell) => cell.imageDigest === digest)).toBe(true) + expect(JSON.stringify(result)).not.toContain('SECRET_TEXT') + }) + + it('represents missing credentials as unknown inventory, never sleeping', async () => { + const gcloud: GcloudClient = { + accessToken: async () => { throw new Error('sensitive context') } + } + let fetchCalls = 0 + const result = await readResourceInventory( + RELAY_OPS_ENVIRONMENTS.production, + gcloud, + async () => { fetchCalls += 1; return Response.json({}) } + ) + expect(fetchCalls).toBe(0) + expect(result.cells.every((cell) => cell.targetSize === null)).toBe(true) + expect(result.cells.every((cell) => cell.backendHealth === 'unknown')).toBe(true) + expect(JSON.stringify(result)).not.toContain('sensitive context') + }) +}) diff --git a/cloud/apps/relay-ops/src/resource-inventory.ts b/cloud/apps/relay-ops/src/resource-inventory.ts new file mode 100644 index 00000000000..62ed3fd862b --- /dev/null +++ b/cloud/apps/relay-ops/src/resource-inventory.ts @@ -0,0 +1,366 @@ +import { z } from 'zod' +import type { RelayOpsEnvironment, RelayOpsCellConfig } from './environment-config.js' +import type { GcloudClient } from './gcloud-client.js' +import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js' + +const RunServiceSchema = z.object({ + template: z.object({ + scaling: z.object({ + minInstanceCount: z.number().optional(), + maxInstanceCount: z.number().optional() + }).optional(), + containers: z.array(z.object({ image: z.string() })).min(1) + }), + conditions: z.array(z.object({ state: z.string() })).default([]), + latestReadyRevision: z.string().optional() +}) + +const SqlInstanceSchema = z.object({ + state: z.string(), + databaseVersion: z.string(), + settings: z.object({ + activationPolicy: z.string(), + availabilityType: z.string().optional(), + tier: z.string() + }) +}) + +const MigSchema = z.object({ + name: z.string(), + targetSize: z.number(), + size: z.union([z.string(), z.number()]).transform(Number).optional(), + instanceGroup: z.string(), + instanceTemplate: z.string(), + status: z.object({ isStable: z.boolean().default(false) }).default({ isStable: false }) +}) + +const TemplateSchema = z.object({ + properties: z.object({ + metadata: z.object({ + items: z.array(z.object({ key: z.string(), value: z.string().optional() })).default([]) + }).optional() + }) +}) + +const BackendHealthGroupSchema = z.object({ + healthStatus: z.array(z.object({ healthState: z.string() })).default([]) +}) +const BackendHealthSchema = z.union([ + BackendHealthGroupSchema, + z.array(z.object({ status: BackendHealthGroupSchema })) +]) + +const CertificateSchema = z.object({ + expireTime: z.string().optional(), + managed: z.object({ + domains: z.array(z.string()).default([]), + state: z.string() + }) +}) + +export type EndpointHealth = { + health: boolean | null + ready: boolean | null + latencyMs: number | null +} + +export type ServiceInventory = { + ready: boolean + revision: string | null + image: string + minInstances: number + maxInstances: number +} + +export type CellInventory = RelayOpsCellConfig & { + migName: string + targetSize: number | null + runningInstances: number | null + stable: boolean | null + template: string | null + imageDigest: string | null + backendHealth: 'healthy' | 'unhealthy' | 'empty' | 'unknown' + endpoint: EndpointHealth +} + +export type ResourceInventory = { + director: ServiceInventory | null + auth: ServiceInventory | null + sql: { + state: string + activationPolicy: string + tier: string + availabilityType: string + databaseVersion: string + } | null + certificate: { state: string; domains: string[]; expireTime: string | null } | null + directorEndpoint: EndpointHealth + authEndpoint: EndpointHealth + cells: CellInventory[] + warnings: string[] +} + +const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null }) +const independentEndpointRetryDelayMs = 11_000 + +function finalSegment(value: string): string { + return value.split('/').at(-1) ?? value +} + +function parseService(value: unknown): ServiceInventory { + const service = RunServiceSchema.parse(value) + return { + ready: service.conditions.length > 0 && service.conditions.every( + (condition) => condition.state === 'CONDITION_SUCCEEDED' + ), + revision: service.latestReadyRevision ? finalSegment(service.latestReadyRevision) : null, + image: service.template.containers[0]!.image, + minInstances: service.template.scaling?.minInstanceCount ?? 0, + maxInstances: service.template.scaling?.maxInstanceCount ?? 0 + } +} + +async function googleRequest( + fetchImpl: typeof fetch, + token: string, + url: string, + init: RequestInit = {} +): Promise { + const response = await fetchImpl(url, { + ...init, + headers: { + authorization: `Bearer ${token}`, + ...(init.body ? { 'content-type': 'application/json' } : {}) + }, + signal: AbortSignal.timeout(30_000) + }) + if (!response.ok) throw new Error(`Google API returned ${response.status}`) + return await response.json() +} + +async function endpointProbe(origin: string, fetchImpl: typeof fetch): Promise { + const startedAt = performance.now() + const check = async (path: '/health' | '/ready'): Promise => { + try { + const response = await fetchImpl(`${origin}${path}`, { + redirect: 'error', + signal: AbortSignal.timeout(8_000) + }) + return response.ok + } catch { + return false + } + } + const [health, ready] = await Promise.all([check('/health'), check('/ready')]) + return { health, ready, latencyMs: Math.round(performance.now() - startedAt) } +} + +export async function probeEndpointHealth( + origin: string, + fetchImpl: typeof fetch, + wait: (ms: number) => Promise = async (ms) => + await new Promise((resolvePromise) => setTimeout(resolvePromise, ms)) +): Promise { + const first = await endpointProbe(origin, fetchImpl) + if ( + first.health && + first.ready && + first.latencyMs !== null && + first.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs + ) { + return first + } + // Outwait Relay's ten-second readiness cache before treating the retry as independent. + await wait(independentEndpointRetryDelayMs) + return await endpointProbe(origin, fetchImpl) +} + +function imageDigest(template: z.infer): string | null { + const startupScript = template.properties.metadata?.items.find( + (item) => item.key === 'startup-script' + )?.value + // Return only the immutable digest; startup metadata contains secret names and operational detail. + return startupScript?.match(/ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'/)?.[1] ?? null +} + +function backendState(value: unknown): CellInventory['backendHealth'] { + const parsedHealth = BackendHealthSchema.parse(value) + const groups = Array.isArray(parsedHealth) + ? parsedHealth.map((group) => group.status) + : [parsedHealth] + const states = groups.flatMap((group) => group.healthStatus.map((status) => status.healthState)) + if (states.length === 0) return 'empty' + if (states.every((state) => state === 'HEALTHY')) return 'healthy' + return 'unhealthy' +} + +function unavailableCell(cell: RelayOpsCellConfig, migName: string): CellInventory { + return { + ...cell, + migName, + targetSize: null, + runningInstances: null, + stable: null, + template: null, + imageDigest: null, + backendHealth: 'unknown', + endpoint: unavailableEndpoint() + } +} + +async function readCell( + environment: RelayOpsEnvironment, + cell: RelayOpsCellConfig, + mig: z.infer | null, + token: string, + fetchImpl: typeof fetch +): Promise { + const migName = `${environment.migPrefix}${cell.hostname}` + if (!mig) return unavailableCell(cell, migName) + // An empty fixed-one MIG cannot serve and must never be woken by observation. + const endpoint = mig.targetSize > 0 + ? await probeEndpointHealth(cell.origin, fetchImpl) + : unavailableEndpoint() + const templateName = finalSegment(mig.instanceTemplate) + const [templateResult, healthResult] = await Promise.allSettled([ + googleRequest( + fetchImpl, + token, + `https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/instanceTemplates/${templateName}` + ), + googleRequest( + fetchImpl, + token, + `https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/backendServices/${migName}/getHealth`, + { method: 'POST', body: JSON.stringify({ group: mig.instanceGroup }) } + ) + ]) + return { + ...cell, + migName, + targetSize: mig.targetSize, + runningInstances: mig.size ?? (mig.status.isStable ? mig.targetSize : null), + stable: mig.status.isStable, + template: templateName, + imageDigest: templateResult.status === 'fulfilled' + ? imageDigest(TemplateSchema.parse(templateResult.value)) + : null, + backendHealth: healthResult.status === 'fulfilled' + ? backendState(healthResult.value) + : 'unknown', + endpoint + } +} + +function parsed( + result: PromiseSettledResult, + schema: S, + warning: string, + warnings: string[] +): z.infer | null { + if (result.status === 'rejected') { + warnings.push(warning) + return null + } + const parsedValue = schema.safeParse(result.value) + if (!parsedValue.success) { + warnings.push(warning) + return null + } + return parsedValue.data +} + +function unavailableInventory(environment: RelayOpsEnvironment, warning: string): ResourceInventory { + return { + director: null, + auth: null, + sql: null, + certificate: null, + directorEndpoint: unavailableEndpoint(), + authEndpoint: unavailableEndpoint(), + cells: environment.cells.map((cell) => + unavailableCell(cell, `${environment.migPrefix}${cell.hostname}`) + ), + warnings: [warning] + } +} + +export async function readResourceInventory( + environment: RelayOpsEnvironment, + gcloud: GcloudClient, + fetchImpl: typeof fetch = fetch +): Promise { + let token: string + try { + token = await gcloud.accessToken() + } catch { + return unavailableInventory( + environment, + 'Google Cloud credentials are unavailable. Run gcloud auth login.' + ) + } + const runUrl = (service: string) => + `https://run.googleapis.com/v2/projects/${environment.project}/locations/${environment.region}/services/${service}` + const migUrl = (cell: RelayOpsCellConfig) => + `https://compute.googleapis.com/compute/v1/projects/${environment.project}/zones/${cell.zone}/instanceGroupManagers/${environment.migPrefix}${cell.hostname}` + const settled = await Promise.allSettled([ + googleRequest(fetchImpl, token, runUrl(environment.directorService)), + googleRequest(fetchImpl, token, runUrl(environment.authService)), + googleRequest( + fetchImpl, + token, + `https://sqladmin.googleapis.com/sql/v1beta4/projects/${environment.project}/instances/${environment.sqlInstance}` + ), + googleRequest( + fetchImpl, + token, + `https://certificatemanager.googleapis.com/v1/projects/${environment.project}/locations/global/certificates/${environment.certificateName}` + ), + ...environment.cells.map((cell) => googleRequest(fetchImpl, token, migUrl(cell))) + ]) + const warnings: string[] = [] + const directorValue = parsed(settled[0]!, RunServiceSchema, 'Director service inventory is unavailable.', warnings) + const authValue = parsed(settled[1]!, RunServiceSchema, 'Auth service inventory is unavailable.', warnings) + const sqlValue = parsed(settled[2]!, SqlInstanceSchema, 'Cloud SQL inventory is unavailable.', warnings) + const certificateValue = parsed( + settled[3]!, CertificateSchema, 'TLS certificate inventory is unavailable.', warnings + ) + const migValues = environment.cells.map((cell, index) => parsed( + settled[index + 4]!, + MigSchema, + `${cell.hostname.toUpperCase()} MIG inventory is unavailable.`, + warnings + )) + const controlPlaneSleeping = + environment.id === 'staging' && sqlValue?.settings.activationPolicy === 'NEVER' + // Health probes would cold-start scale-to-zero Cloud Run services, so sleeping staging is inventory-only. + const [directorEndpoint, authEndpoint] = controlPlaneSleeping + ? [unavailableEndpoint(), unavailableEndpoint()] + : await Promise.all([ + probeEndpointHealth(environment.directorOrigin, fetchImpl), + probeEndpointHealth(environment.authOrigin, fetchImpl) + ]) + const cells = await Promise.all(environment.cells.map((cell, index) => + readCell(environment, cell, migValues[index] ?? null, token, fetchImpl) + )) + return { + director: directorValue ? parseService(directorValue) : null, + auth: authValue ? parseService(authValue) : null, + sql: sqlValue ? { + state: sqlValue.state, + activationPolicy: sqlValue.settings.activationPolicy, + tier: sqlValue.settings.tier, + availabilityType: sqlValue.settings.availabilityType ?? 'unknown', + databaseVersion: sqlValue.databaseVersion + } : null, + certificate: certificateValue ? { + state: certificateValue.managed.state, + domains: certificateValue.managed.domains, + expireTime: certificateValue.expireTime ?? null + } : null, + directorEndpoint, + authEndpoint, + cells, + warnings + } +} diff --git a/cloud/apps/relay-ops/src/staging-workflow.test.ts b/cloud/apps/relay-ops/src/staging-workflow.test.ts new file mode 100644 index 00000000000..881ffb2bf9a --- /dev/null +++ b/cloud/apps/relay-ops/src/staging-workflow.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' +import { parseStagingPowerRequest } from './staging-workflow.js' + +describe('parseStagingPowerRequest', () => { + it('accepts the exact reviewed confirmations', () => { + expect(parseStagingPowerRequest({ mode: 'status', confirmation: '' })).toEqual({ + mode: 'status', confirmation: '' + }) + expect(parseStagingPowerRequest({ mode: 'wake', confirmation: 'WAKE_STAGING' }).mode).toBe('wake') + expect(parseStagingPowerRequest({ mode: 'sleep', confirmation: 'SLEEP_STAGING' }).mode).toBe('sleep') + }) + + it('rejects missing, swapped, or additional fields', () => { + expect(() => parseStagingPowerRequest({ mode: 'wake', confirmation: '' })).toThrow() + expect(() => parseStagingPowerRequest({ mode: 'sleep', confirmation: 'WAKE_STAGING' })).toThrow() + expect(() => parseStagingPowerRequest({ mode: 'production', confirmation: '' })).toThrow() + expect(() => parseStagingPowerRequest({ mode: 'status', confirmation: '', project: 'other' })).toThrow() + }) +}) diff --git a/cloud/apps/relay-ops/src/staging-workflow.ts b/cloud/apps/relay-ops/src/staging-workflow.ts new file mode 100644 index 00000000000..749c6e865bb --- /dev/null +++ b/cloud/apps/relay-ops/src/staging-workflow.ts @@ -0,0 +1,36 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { z } from 'zod' +import { RELAY_GITHUB_REPOSITORY, relayWorkflowFile } from './relay-repository.js' + +const execFileAsync = promisify(execFile) +const DispatchSchema = z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('status'), confirmation: z.literal('') }).strict(), + z.object({ mode: z.literal('wake'), confirmation: z.literal('WAKE_STAGING') }).strict(), + z.object({ mode: z.literal('sleep'), confirmation: z.literal('SLEEP_STAGING') }).strict() +]) + +export type StagingPowerRequest = z.infer + +export function parseStagingPowerRequest(value: unknown): StagingPowerRequest { + return DispatchSchema.parse(value) +} + +export async function dispatchStagingPowerWorkflow(request: StagingPowerRequest): Promise { + const args = [ + 'workflow', 'run', relayWorkflowFile('power-relay-staging.yml'), + '--repo', RELAY_GITHUB_REPOSITORY, + '-f', `mode=${request.mode}`, + '-f', 'wake-cells=configured' + ] + if (request.confirmation) args.push('-f', `confirmation=${request.confirmation}`) + try { + await execFileAsync('gh', args, { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 1024 * 1024 + }) + } catch { + throw new Error('Staging power workflow dispatch failed') + } +} diff --git a/cloud/apps/relay-ops/tsconfig.build.json b/cloud/apps/relay-ops/tsconfig.build.json new file mode 100644 index 00000000000..0a11719fef1 --- /dev/null +++ b/cloud/apps/relay-ops/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["src/**/*.test.ts"] +} diff --git a/cloud/apps/relay-ops/tsconfig.json b/cloud/apps/relay-ops/tsconfig.json new file mode 100644 index 00000000000..d266baf8ad6 --- /dev/null +++ b/cloud/apps/relay-ops/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "types": ["node", "vitest"], + "exactOptionalPropertyTypes": true, + "verbatimModuleSyntax": true + }, + "include": ["src/**/*.ts"] +} diff --git a/cloud/apps/relay/Dockerfile b/cloud/apps/relay/Dockerfile new file mode 100644 index 00000000000..12516cbf749 --- /dev/null +++ b/cloud/apps/relay/Dockerfile @@ -0,0 +1,25 @@ +FROM node:24-alpine AS build +WORKDIR /app +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./ +COPY packages/relay-contract/package.json packages/relay-contract/package.json +COPY apps/relay/package.json apps/relay/package.json +RUN pnpm install --frozen-lockfile +COPY packages/relay-contract packages/relay-contract +COPY apps/relay apps/relay +RUN pnpm --filter @orca-cloud/relay-contract build && pnpm --filter @orca-cloud/relay build + +FROM node:24-alpine AS runtime +ENV NODE_ENV=production +ENV PORT=8080 +WORKDIR /app +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY packages/relay-contract/package.json packages/relay-contract/package.json +COPY apps/relay/package.json apps/relay/package.json +COPY --from=build /app/packages/relay-contract/dist packages/relay-contract/dist +COPY --from=build /app/apps/relay/dist apps/relay/dist +RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay... +USER node +EXPOSE 8080 +CMD ["node", "apps/relay/dist/index.js"] diff --git a/cloud/apps/relay/package.json b/cloud/apps/relay/package.json new file mode 100644 index 00000000000..4c2b2e4269c --- /dev/null +++ b/cloud/apps/relay/package.json @@ -0,0 +1,35 @@ +{ + "name": "@orca-cloud/relay", + "private": true, + "version": "0.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "pnpm clean && tsc -p tsconfig.build.json", + "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", + "dev": "tsx watch src/index.ts", + "lint": "tsc -p tsconfig.json --noEmit", + "pretest": "pnpm --filter @orca-cloud/relay-contract build", + "start": "node dist/index.js", + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "@hono/node-server": "^1.19.14", + "@orca-cloud/relay-contract": "workspace:*", + "hono": "^4.12.27", + "jose": "^6.1.3", + "pg": "^8.22.0", + "tweetnacl": "^1.0.3", + "ws": "^8.18.3", + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "@types/pg": "^8.20.0", + "@types/ws": "^8.18.1", + "tsx": "^4.21.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/apps/relay/src/admin-token-verifier.test.ts b/cloud/apps/relay/src/admin-token-verifier.test.ts new file mode 100644 index 00000000000..f0870dbb09c --- /dev/null +++ b/cloud/apps/relay/src/admin-token-verifier.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' +import { + RELAY_ASIA_PROOF_ADMIN_ROUTES, + RELAY_CAPACITY_ADMIN_ROUTES, + RELAY_FENCE_BROKER_ADMIN_ROUTES, + RELAY_FENCE_ADMIN_ROUTES, + RELAY_MONITOR_ADMIN_ROUTES, + relayAdminIdentityMayAccess +} from './admin-token-verifier.js' + +const mutationRoutes = [ + '/v1/admin/drain', + '/v1/admin/evacuate', + '/v1/admin/migration-complete', + '/v1/admin/migration-supersede-cell', + '/v1/admin/rebalance-dormant', + '/v1/admin/admission-selector/apply', + '/v1/admin/admission-selector/add-migration-cells', + '/v1/admin/cell-state', + '/v1/admin/cell-fence-adopt-legacy', + '/v1/admin/cell-fence-commit-legacy-adoption', + '/v1/admin/cell-fence-attest', + '/v1/admin/cell-fence-attempt-prepare', + '/v1/admin/cell-fence-attempt-start', + '/v1/admin/cell-fence-attempt-operation', + '/v1/admin/cell-fence-attempt-abort', + '/v1/admin/drain-attempt-prepare', + '/v1/admin/drain-attempt-send', + '/v1/admin/drain-attempt-receipt', + '/v1/admin/drain-attempt-recover-forward', + '/v1/admin/cell-config', + '/v1/admin/evacuate-cell', + '/v1/admin/cell-heartbeat', + '/v1/admin/regional-rehome-control', + '/v1/admin/regional-rehome-trust-probe' +] as const + +describe('Relay admin route authorization', () => { + it('allows the staging capacity identity only its transition routes', () => { + for (const route of RELAY_CAPACITY_ADMIN_ROUTES) { + expect(relayAdminIdentityMayAccess('capacity', route)).toBe(true) + } + for (const route of mutationRoutes) { + if ((RELAY_CAPACITY_ADMIN_ROUTES as readonly string[]).includes(route)) continue + expect(relayAdminIdentityMayAccess('capacity', route)).toBe(false) + } + expect(RELAY_CAPACITY_ADMIN_ROUTES).toContain('/v1/admin/cell-state') + expect(relayAdminIdentityMayAccess('capacity', '/v1/admin/evacuation-status')).toBe(false) + }) + + it('allows the Asia proof identity only its selector and status routes', () => { + for (const route of RELAY_ASIA_PROOF_ADMIN_ROUTES) { + expect(relayAdminIdentityMayAccess('asia-proof', route)).toBe(true) + } + expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/drain')).toBe(false) + expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/cell-state')).toBe(false) + expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/admission-selector/apply')).toBe(false) + expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/add-migration-cells')).toBe(false) + }) + + it('keeps the monitor identity on exact aggregate read routes', () => { + for (const route of RELAY_MONITOR_ADMIN_ROUTES) { + expect(relayAdminIdentityMayAccess('monitor', route)).toBe(true) + } + for (const route of [...mutationRoutes, ...RELAY_FENCE_ADMIN_ROUTES]) { + if ((RELAY_MONITOR_ADMIN_ROUTES as readonly string[]).includes(route)) continue + expect(relayAdminIdentityMayAccess('monitor', route)).toBe(false) + } + expect(RELAY_MONITOR_ADMIN_ROUTES).toContain('/v1/admin/evacuation-status') + }) + + it('allows only reviewed fence evidence mutations beyond aggregate reads', () => { + for (const route of RELAY_FENCE_ADMIN_ROUTES) { + expect(relayAdminIdentityMayAccess('fence', route)).toBe(true) + } + for (const route of mutationRoutes) { + if ((RELAY_FENCE_ADMIN_ROUTES as readonly string[]).includes(route)) continue + expect(relayAdminIdentityMayAccess('fence', route)).toBe(false) + } + }) + + it('allows the broker only exact fence inspection and mutation routes', () => { + for (const route of RELAY_FENCE_BROKER_ADMIN_ROUTES) { + expect(relayAdminIdentityMayAccess('fence-broker', route)).toBe(true) + } + for (const route of [...mutationRoutes, ...RELAY_FENCE_ADMIN_ROUTES]) { + if ((RELAY_FENCE_BROKER_ADMIN_ROUTES as readonly string[]).includes(route)) continue + expect(relayAdminIdentityMayAccess('fence-broker', route)).toBe(false) + } + }) + + it('rejects unknown routes for dedicated identities', () => { + for (const identity of ['capacity', 'asia-proof', 'monitor', 'fence', 'fence-broker'] as const) { + expect(relayAdminIdentityMayAccess(identity, '/v1/admin/future-mutation')).toBe(false) + expect(relayAdminIdentityMayAccess(identity, '/health')).toBe(false) + } + }) +}) diff --git a/cloud/apps/relay/src/admin-token-verifier.ts b/cloud/apps/relay/src/admin-token-verifier.ts new file mode 100644 index 00000000000..4b8ad26e695 --- /dev/null +++ b/cloud/apps/relay/src/admin-token-verifier.ts @@ -0,0 +1,202 @@ +import { createRemoteJWKSet, jwtVerify } from 'jose' +import type { RelayConfig } from './config.js' + +export const RELAY_MONITOR_ADMIN_ROUTES = [ + '/v1/admin/admission-selector/status', + '/v1/admin/cell-status', + '/v1/admin/evacuation-status', + '/v1/admin/regional-rehome-control', + '/v1/admin/runtime-status' +] as const + +export const RELAY_FENCE_ADMIN_ROUTES = [ + ...RELAY_MONITOR_ADMIN_ROUTES, + '/v1/admin/evacuation-capacity', + '/v1/admin/cell-fence-attempt-status' +] as const + +export const RELAY_CAPACITY_ADMIN_ROUTES = [ + '/v1/admin/admission-selector/status', + '/v1/admin/admission-selector/apply', + '/v1/admin/cell-state', + '/v1/admin/cell-status', + '/v1/admin/runtime-status', + '/v1/admin/drain' +] as const + +export const RELAY_ASIA_PROOF_ADMIN_ROUTES = [ + '/v1/admin/admission-selector/status', + '/v1/admin/admission-selector/apply-staging-asia-proof', + '/v1/admin/cell-status', + '/v1/admin/runtime-status' +] as const + +export const RELAY_FENCE_BROKER_ADMIN_ROUTES = [ + ...RELAY_FENCE_ADMIN_ROUTES, + '/v1/admin/cell-fence-adopt-legacy', + '/v1/admin/cell-fence-commit-legacy-adoption', + '/v1/admin/cell-fence-attest', + '/v1/admin/cell-fence-attempt-prepare', + '/v1/admin/cell-fence-attempt-start', + '/v1/admin/cell-fence-attempt-plan', + '/v1/admin/cell-fence-attempt-operation', + '/v1/admin/cell-fence-attempt-abort', + '/v1/admin/migration-supersede-cell' +] as const + +type RelayAdminIdentity = + | 'deploy' + | 'capacity' + | 'asia-proof' + | 'monitor' + | 'fence' + | 'fence-broker' + +function createGoogleServiceTokenVerifier(input: { + jwksUrl: string + audience: string + serviceAccount: string +}): (token: string) => Promise { + const jwks = createRemoteJWKSet(new URL(input.jwksUrl)) + return async (token) => { + try { + const { payload } = await jwtVerify(token, jwks, { + issuer: ['https://accounts.google.com', 'accounts.google.com'], + audience: input.audience, + algorithms: ['RS256'] + }) + return payload.email === input.serviceAccount && payload.email_verified === true + } catch { + return false + } + } +} + +function createGoogleServiceTokenIdentityVerifier(input: { + jwksUrl: string + audience: string + serviceAccounts: ReadonlyMap +}): (token: string) => Promise { + const jwks = createRemoteJWKSet(new URL(input.jwksUrl)) + return async (token) => { + try { + const { payload } = await jwtVerify(token, jwks, { + issuer: ['https://accounts.google.com', 'accounts.google.com'], + audience: input.audience, + algorithms: ['RS256'] + }) + if (payload.email_verified !== true || typeof payload.email !== 'string') return null + return input.serviceAccounts.get(payload.email) ?? null + } catch { + return null + } + } +} + +export function relayAdminIdentityMayAccess( + identity: RelayAdminIdentity, + route: string +): boolean { + if (identity === 'deploy') return route.startsWith('/v1/admin/') + if (identity === 'capacity') { + return (RELAY_CAPACITY_ADMIN_ROUTES as readonly string[]).includes(route) + } + if (identity === 'asia-proof') { + return (RELAY_ASIA_PROOF_ADMIN_ROUTES as readonly string[]).includes(route) + } + if (identity === 'monitor') { + return (RELAY_MONITOR_ADMIN_ROUTES as readonly string[]).includes(route) + } + if (identity === 'fence') { + return (RELAY_FENCE_ADMIN_ROUTES as readonly string[]).includes(route) + } + return (RELAY_FENCE_BROKER_ADMIN_ROUTES as readonly string[]).includes(route) +} + +export function createReadOnlyAdminTokenVerifier( + config: RelayConfig +): (token: string) => Promise { + const serviceAccounts = new Map() + if (config.monitorServiceAccount) serviceAccounts.set(config.monitorServiceAccount, 'monitor') + if (config.fenceServiceAccount) serviceAccounts.set(config.fenceServiceAccount, 'fence') + if (serviceAccounts.size === 0) return async () => false + const verifyIdentity = createGoogleServiceTokenIdentityVerifier({ + jwksUrl: config.adminJwksUrl, + audience: config.adminAudience, + serviceAccounts + }) + return async (token) => (await verifyIdentity(token)) !== null +} + +export function createAdminTokenVerifier( + config: RelayConfig +): (token: string, route?: string) => Promise { + const serviceAccounts = new Map([ + [config.deployServiceAccount, 'deploy'] + ]) + if (config.capacityServiceAccount) { + serviceAccounts.set(config.capacityServiceAccount, 'capacity') + } + if (config.asiaProofServiceAccount) { + serviceAccounts.set(config.asiaProofServiceAccount, 'asia-proof') + } + if (config.monitorServiceAccount) serviceAccounts.set(config.monitorServiceAccount, 'monitor') + if (config.fenceServiceAccount) serviceAccounts.set(config.fenceServiceAccount, 'fence') + if (config.fenceBrokerServiceAccount) { + serviceAccounts.set(config.fenceBrokerServiceAccount, 'fence-broker') + } + const verifyIdentity = createGoogleServiceTokenIdentityVerifier({ + jwksUrl: config.adminJwksUrl, + audience: config.adminAudience, + serviceAccounts + }) + return async (token, route) => { + const identity = await verifyIdentity(token) + return identity !== null && (!route || relayAdminIdentityMayAccess(identity, route)) + } +} + +export function createRegionalRehomeControlApplyTokenVerifier( + config: RelayConfig +): (token: string) => Promise { + return createGoogleServiceTokenVerifier({ + jwksUrl: config.adminJwksUrl, + audience: config.adminAudience, + serviceAccount: config.deployServiceAccount + }) +} + +export function createRuntimeTokenVerifier( + config: RelayConfig +): (token: string) => Promise { + if (!config.heartbeatAudience) return async () => false + return createGoogleServiceTokenVerifier({ + jwksUrl: config.adminJwksUrl, + audience: config.heartbeatAudience, + serviceAccount: config.runtimeServiceAccount + }) +} + +export function createRegionalRehomeTokenVerifier( + config: RelayConfig +): (token: string) => Promise { + if (!config.rehomeAudience || !config.rehomeDirectorServiceAccount) { + return async () => false + } + return createGoogleServiceTokenVerifier({ + jwksUrl: config.adminJwksUrl, + audience: config.rehomeAudience, + serviceAccount: config.rehomeDirectorServiceAccount + }) +} + +export function createRegionalRehomeRuntimeTokenVerifier( + config: RelayConfig +): (token: string) => Promise { + if (!config.rehomeAudience) return async () => false + return createGoogleServiceTokenVerifier({ + jwksUrl: config.adminJwksUrl, + audience: config.rehomeAudience, + serviceAccount: config.runtimeServiceAccount + }) +} diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts new file mode 100644 index 00000000000..3df01d9e9ce --- /dev/null +++ b/cloud/apps/relay/src/app.ts @@ -0,0 +1,1850 @@ +import { + AssignmentRequestSchema, + isRelayCellConnectionHardCap, + RELAY_ADMISSION_BUDGETS, + RELAY_DEFAULT_REGION, + RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND, + RELAY_PROTOCOL_LIMITS, + RelayRegionSchema, + ResolveRequestSchema, + cellPlacementCeiling, + relayCellAdmissionBounds, + type RelayCellConnectionHardCap, + type RelayRegion +} from '@orca-cloud/relay-contract' +import { Hono, type Context } from 'hono' +import { SignJWT } from 'jose' +import { z } from 'zod' +import { + createAdminTokenVerifier, + createReadOnlyAdminTokenVerifier, + createRegionalRehomeControlApplyTokenVerifier, + createRegionalRehomeRuntimeTokenVerifier, + createRegionalRehomeTokenVerifier, + createRuntimeTokenVerifier +} from './admin-token-verifier.js' +import type { + CellFenceAttemptEvidence, + RelayAssignment, + RelayAssignmentStore +} from './assignment-store.js' +import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js' +import { CELL_ADMISSION_STATES } from './cell-admission-selector.js' +import { RELAY_MAX_CELL_CAPACITY_REQUESTS, type RelayConfig } from './config.js' +import type { RelayCredentialStore } from './credential-store.js' +import { isRelayDatabaseTransientError } from './database.js' +import { googleMetadataIdentityToken } from './google-metadata-identity-token.js' +import { + RelayPublicAssignmentAdmission, + type AssignmentAdmissionRejection +} from './public-assignment-admission.js' +import { relayHostLogDigest } from './relay-host-log-digest.js' +import type { RelayRuntimeCounts } from './relay-observability.js' +import { + isRegionalRehomeTrustProbe, + probeRegionalRehomeTrust +} from './regional-rehome-trust-probe.js' +import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js' +import { stagingAsiaProofMembership } from './staging-asia-proof-admission.js' + +const RelayCellConnectionHardCapSchema = z.custom( + isRelayCellConnectionHardCap +) + +const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000 +const REGION_CATALOG_CACHE_MS = 30_000 + +type AdmissionRejectionLogEntry = { + route: 'assign' | 'resolve' + lane: 'sticky' | 'placement' + hinted: boolean + relayHostId: string + reason: AssignmentAdmissionRejection +} + +export function createRelayApp( + config: RelayConfig, + operations: { + store: RelayCredentialStore + assignments: RelayAssignmentStore + drain: (graceMs: number) => void + drainHost?: (input: { + attemptId: string + userId: string + relayHostId: string + sourceAssignmentEpoch: number + graceMs: number + }) => 'accepted' | 'already-accepted' | 'host-not-connected' + regionalRehomeIdentityToken?: (audience: string) => Promise + regionalRehomeFetch?: typeof fetch + regionalRehomeTrustProbeHostExists?: (input: { + userId: string + relayHostId: string + }) => boolean + cellIncarnation?: string + isDraining?: () => boolean + runtimeCounts?: () => RelayRuntimeCounts + ready: () => Promise + recordAssignmentAdmission?: ( + outcome: 'sticky' | 'sticky-rejected' | 'placement' | 'placement-rejected' + ) => void + recordAssignmentRejectionReason?: ( + lane: 'sticky' | 'placement', + reason: AssignmentAdmissionRejection + ) => void + recordRegionRequest?: (region: RelayRegion | undefined) => void + recordRegionSelection?: (input: { + targetRegion: RelayRegion + selectedRegion?: RelayRegion + fallback: boolean + }) => void + } +): Hono { + const app = new Hono() + let regionCatalogCache: + | { expiresAt: number; value: Awaited> } + | undefined + let regionCatalogRefresh: + | Promise>> + | undefined + const regionCatalog = async () => { + const now = Date.now() + if (regionCatalogCache && regionCatalogCache.expiresAt > now) { + return regionCatalogCache.value + } + regionCatalogRefresh ??= operations.assignments.regionCatalog().then((value) => { + regionCatalogCache = { expiresAt: Date.now() + REGION_CATALOG_CACHE_MS, value } + return value + }) + try { + return await regionCatalogRefresh + } finally { + regionCatalogRefresh = undefined + } + } + const verifyRelayToken = createRelayTokenVerifier(config) + const verifyAdminToken = createAdminTokenVerifier(config) + const verifyReadOnlyAdminToken = createReadOnlyAdminTokenVerifier(config) + const verifyRegionalRehomeControlApplyToken = + createRegionalRehomeControlApplyTokenVerifier(config) + const verifyRuntimeToken = createRuntimeTokenVerifier(config) + const verifyRegionalRehomeToken = createRegionalRehomeTokenVerifier(config) + const verifyRegionalRehomeRuntimeToken = + createRegionalRehomeRuntimeTokenVerifier(config) + const regionalRehomeFetch = operations.regionalRehomeFetch ?? fetch + const regionalRehomeIdentityToken = + operations.regionalRehomeIdentityToken ?? + ((audience: string) => googleMetadataIdentityToken(audience, regionalRehomeFetch)) + const publicAssignmentAdmission = new RelayPublicAssignmentAdmission({ + maxConcurrent: config.publicAssignmentConcurrency, + maxQueued: config.publicAssignmentQueueMax, + waitMs: config.publicAssignmentWaitMs, + maxReservedConcurrent: config.publicResolveConcurrency, + reservedWaitMs: config.publicResolveWaitMs, + minIntervalMs: config.publicAssignmentRetryAfterSeconds * 1_000, + onRejected: (reason) => operations.recordAssignmentRejectionReason?.('placement', reason) + }) + const rejectPublicAssignment = (context: Context): Response => { + context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds)) + return context.json({ error: 'assignments_temporarily_unavailable' }, 503) + } + // Reconnecting hosts declare themselves and are verified against the durable + // assignment inside this bounded lane, so a placement backlog can never + // starve session recovery. Unhinted traffic never touches this lane. + const stickyRetryAfterSeconds = config.publicStickyRetryAfterSeconds ?? 2 + const stickyAssignmentAdmission = new RelayPublicAssignmentAdmission({ + maxConcurrent: config.publicStickyConcurrency ?? 1, + maxQueued: config.publicStickyQueueMax ?? 64, + waitMs: config.publicStickyWaitMs ?? 2_000, + minIntervalMs: stickyRetryAfterSeconds * 1_000, + onRejected: (reason) => operations.recordAssignmentRejectionReason?.('sticky', reason) + }) + const rejectStickyAssignment = (context: Context): Response => { + context.header('Retry-After', String(stickyRetryAfterSeconds)) + return context.json({ error: 'assignments_temporarily_unavailable' }, 503) + } + // Aggregate counters cannot separate a handful of pathological hosts from a broad + // population, so every admission rejection names its host and reason. Keyed on + // route:lane:reason rather than host, the log stays bounded under load. + const rejectionLogWindow = new AssignmentRejectionLogWindow({ + windowMs: ASSIGNMENT_REJECTION_LOG_WINDOW_MS, + onWindowClosed: ({ suppressed, sample }) => logAssignmentRejection({ ...sample, suppressed }) + }) + const logAdmissionRejection = (input: { + route: 'assign' | 'resolve' + lane: 'sticky' | 'placement' + hinted: boolean + relayHostId: string + reason: AssignmentAdmissionRejection | undefined + }): void => { + if (!input.reason) return + const entry: AdmissionRejectionLogEntry = { ...input, reason: input.reason } + if (!rejectionLogWindow.admit(`${entry.route}:${entry.lane}:${entry.reason}`, entry)) return + logAssignmentRejection(entry) + } + app.use('/v1/admin/*', async (context, next) => { + if ( + context.req.path === '/v1/admin/cell-heartbeat' || + context.req.path === '/v1/admin/cell-rehome-status' + ) { + return await next() + } + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyAdminToken(bearer))) return await next() + if (!(await verifyAdminToken(bearer, context.req.path))) { + return context.json({ error: 'invalid_token' }, 401) + } + return await next() + }) + + // Not /healthz: Google Front End reserves that path before the container. + app.get('/health', (context) => + context.json({ ok: true, connectionCapacityProtocol: 2 }) + ) + app.get('/ready', async (context) => + (await operations.ready()) + ? context.json({ ok: true }) + : context.json({ error: 'dependency_unavailable' }, 503) + ) + app.get('/v1/regions', async (context) => { + if (config.role === 'cell') return context.json({ error: 'director_only' }, 404) + return context.json({ v: 1, regions: await regionCatalog() }) + }) + app.post('/v1/assign', async (context) => { + if (config.role === 'cell') return context.json({ error: 'director_only' }, 404) + if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context) + const bearer = readBearer(context.req.header('authorization')) + if (!bearer) return context.json({ error: 'invalid_token' }, 401) + const claims = await verifyRelayToken(bearer) + if (!claims) return context.json({ error: 'invalid_token' }, 401) + if (Number(context.req.header('content-length') ?? 0) > 4 * 1024) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AssignmentRequestSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + if (body.data.relayHostId !== claims.relayHostId) { + return context.json({ error: 'host_identity_mismatch' }, 403) + } + const identity = { userId: claims.sub, relayHostId: claims.relayHostId } + const requestedRegion = body.data.preferredRegion + const targetRegion = + config.regionalPlacementEnabled !== false && requestedRegion + ? requestedRegion + : RELAY_DEFAULT_REGION + operations.recordRegionRequest?.(requestedRegion) + let admission: { release(): void } | null = null + let lane: 'sticky' | 'placement' = 'placement' + if (body.data.reconnect) { + let rejection: AssignmentAdmissionRejection | undefined + const fastLane = await stickyAssignmentAdmission.acquire(claims.relayHostId, (reason) => { + rejection = reason + }) + if (!fastLane) { + operations.recordAssignmentAdmission?.('sticky-rejected') + logAdmissionRejection({ + route: 'assign', + lane: 'sticky', + hinted: true, + relayHostId: claims.relayHostId, + reason: rejection + }) + return rejectStickyAssignment(context) + } + let verified = false + try { + verified = (await operations.assignments.resolve(identity)) !== null + } catch (error) { + fastLane.release() + operations.recordAssignmentAdmission?.('sticky-rejected') + if (isRelayDatabaseTransientError(error)) { + logAssignmentRejection({ + route: 'assign-verify', + lane: 'none', + hinted: true, + relayHostId: claims.relayHostId, + reason: operationError(error) + }) + return rejectStickyAssignment(context) + } + throw error + } + if (verified) { + lane = 'sticky' + admission = fastLane + operations.recordAssignmentAdmission?.('sticky') + } else { + // An unverified hint joins the placement lane with today's semantics. + fastLane.release() + } + } + if (!admission) { + let rejection: AssignmentAdmissionRejection | undefined + admission = await publicAssignmentAdmission.acquire(claims.relayHostId, (reason) => { + rejection = reason + }) + operations.recordAssignmentAdmission?.(admission ? 'placement' : 'placement-rejected') + if (!admission) { + logAdmissionRejection({ + route: 'assign', + lane: 'placement', + hinted: Boolean(body.data.reconnect), + relayHostId: claims.relayHostId, + reason: rejection + }) + return rejectPublicAssignment(context) + } + } + let assignment: RelayAssignment + try { + assignment = requestedRegion + ? await operations.assignments.assign(identity, requestedRegion, targetRegion) + : await operations.assignments.assign(identity) + } catch (error) { + if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) { + logAssignmentRejection({ + route: 'assign', + lane, + hinted: Boolean(body.data.reconnect), + relayHostId: claims.relayHostId, + reason: operationError(error) + }) + } + if (isRelayAssignmentCapacityError(error)) { + if (lane === 'placement') { + operations.recordRegionSelection?.({ targetRegion, fallback: false }) + } + return context.json({ error: operationError(error) }, 503) + } + if (isRelayDatabaseTransientError(error)) { + return lane === 'sticky' ? rejectStickyAssignment(context) : rejectPublicAssignment(context) + } + throw error + } finally { + admission.release() + } + operations.recordRegionSelection?.({ + targetRegion, + selectedRegion: assignment.region, + fallback: lane === 'placement' && assignment.region !== targetRegion + }) + // Grant-side counterpart of the rejection log: reconnect grants are rare + // enough to log and make "which cell is this host on" answerable. + if (lane === 'sticky') { + console.warn( + `[orca-relay] assignment granted lane=sticky host=${relayHostLogDigest(claims.relayHostId)}` + + ` cell=${assignment.cellId}` + ) + } + const lease = await new SignJWT({ + purpose: 'cell-assignment', + cellId: assignment.cellId, + cellUrl: assignment.cellUrl, + assignmentEpoch: assignment.assignmentEpoch, + relayHostId: claims.relayHostId + }) + .setProtectedHeader({ alg: 'HS256' }) + .setIssuer(config.publicUrl) + .setAudience('orca-relay-cell') + .setSubject(claims.sub) + .setIssuedAt() + .setExpirationTime('5m') + .sign(config.assignmentSigningKey) + return context.json({ + v: 1, + cellUrl: assignment.cellUrl, + assignmentEpoch: assignment.assignmentEpoch, + lease + }) + }) + app.post('/v1/resolve', async (context) => { + if (config.role === 'cell') return context.json({ error: 'director_only' }, 404) + if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context) + if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + let rejection: AssignmentAdmissionRejection | undefined + const admission = await publicAssignmentAdmission.acquireReserved( + body.data.relayHostId, + (reason) => { + rejection = reason + } + ) + if (!admission) { + logAdmissionRejection({ + route: 'resolve', + lane: 'placement', + hinted: false, + relayHostId: body.data.relayHostId, + reason: rejection + }) + return rejectPublicAssignment(context) + } + try { + const resolved = await operations.store.resolveResume( + body.data.relayHostId, + body.data.resumeToken + ) + if (!resolved) return context.json({ error: 'invalid_credential' }, 401) + const identity = { + userId: resolved.userId, + relayHostId: body.data.relayHostId + } + // This also migrates credentials created by the staging-only combined service. + const assignment = + (await operations.assignments.resolve(identity)) ?? + (await operations.assignments.assign(identity)) + return context.json({ + v: 1, + cellUrl: assignment.cellUrl, + assignmentEpoch: assignment.assignmentEpoch, + leaseExpiresAt: assignment.leaseExpiresAt + }) + } catch (error) { + if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) { + logAssignmentRejection({ + route: 'resolve', + lane: 'none', + hinted: false, + relayHostId: body.data.relayHostId, + reason: operationError(error) + }) + } + if (isRelayAssignmentCapacityError(error)) { + return context.json({ error: operationError(error) }, 503) + } + if (isRelayDatabaseTransientError(error)) return rejectPublicAssignment(context) + throw error + } finally { + admission.release() + } + }) + app.post('/v1/admin/drain', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + const body = z + .object({ v: z.literal(1), graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) }) + .strict() + .safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + operations.drain(body.data.graceMs) + return context.json({ ok: true }) + }) + app.post('/v1/admin/host-drain', async (context) => { + if (config.role !== 'cell' || !operations.drainHost) { + return context.json({ error: 'cell_only' }, 404) + } + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyRegionalRehomeToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = RegionalHostDrainSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + if ( + body.data.sourceCellId !== config.cellId || + !operations.cellIncarnation || + body.data.sourceCellIncarnation !== operations.cellIncarnation + ) { + return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409) + } + try { + const trustProbe = isRegionalRehomeTrustProbe(body.data) + let sharedRuntimeIdentityRejected: true | undefined + if (trustProbe) { + const runtimeToken = await regionalRehomeIdentityToken(config.rehomeAudience!) + if ( + !(await verifyRegionalRehomeRuntimeToken(runtimeToken)) || + (await verifyRegionalRehomeToken(runtimeToken)) + ) { + throw new Error('regional_rehome_shared_runtime_rejection_not_proven') + } + if ( + !operations.regionalRehomeTrustProbeHostExists || + operations.regionalRehomeTrustProbeHostExists(body.data) + ) { + throw new Error('regional_rehome_trust_probe_host_not_absent') + } + sharedRuntimeIdentityRejected = true + } + const outcome = operations.drainHost(body.data) + return context.json({ + v: 1, + outcome, + ...(sharedRuntimeIdentityRejected ? { sharedRuntimeIdentityRejected } : {}) + }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/runtime-status', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = RuntimeStatusSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + return context.json({ + v: 1, + role: config.role, + cellId: config.cellId, + cellUrl: config.cellUrl, + region: config.region ?? RELAY_DEFAULT_REGION, + imageDigest: config.imageDigest ?? null, + draining: operations.isDraining?.() ?? false, + regionalRehomeProtocol: + config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0, + connectionCapacity: + config.connectionHardCap === undefined + ? null + : { + hardCap: config.connectionHardCap, + controlRebindReserve: RELAY_ADMISSION_BUDGETS.reservedHostControls, + ordinaryConnectionLimit: relayCellAdmissionBounds(config.connectionHardCap) + .socketAdmissionCeiling, + unobservedBound: config.connectionUnobservedBound!, + normalAdmissionPause: cellPlacementCeiling( + config.connectionHardCap, + config.connectionUnobservedBound! + ) + }, + runtime: operations.runtimeCounts?.() ?? null + }) + }) + app.post('/v1/admin/cell-heartbeat', async (context) => { + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyRuntimeToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = CellHeartbeatSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + await operations.assignments.recordCellHeartbeat(body.data) + return context.json({ ok: true }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-rehome-status', async (context) => { + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyRuntimeToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = CellRegionalRehomeStatusSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + await operations.assignments.recordCellRegionalRehomeStatus(body.data) + return context.json({ ok: true }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/regional-rehome-control', async (context) => { + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = RegionalRehomeControlSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + if (body.data.action === 'inspect') { + const control = await operations.assignments.inspectRegionalRehomeControl() + return context.json({ v: 1, control }) + } + if (!(await verifyRegionalRehomeControlApplyToken(bearer))) { + return context.json({ error: 'insufficient_permission' }, 403) + } + try { + const control = await operations.assignments.applyRegionalRehomeControl(body.data) + return context.json({ v: 1, control }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/regional-rehome-trust-probe', async (context) => { + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = RegionalRehomeTrustProbeSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + if (!config.rehomeAudience || !config.rehomeDirectorServiceAccount) { + return context.json({ error: 'regional_rehome_trust_not_configured' }, 409) + } + try { + const source = await operations.assignments.cellDeploymentStatus( + body.data.sourceCellId + ) + if ( + source.region !== RELAY_DEFAULT_REGION || + !source.runtime || + source.runtime.cellIncarnation !== body.data.sourceCellIncarnation || + !source.runtime.ready || + !source.runtime.heartbeatFresh || + source.runtime.regionalRehomeProtocol < 1 + ) { + throw new Error('regional_rehome_trust_probe_source_unavailable') + } + const result = await probeRegionalRehomeTrust({ + sourceCellUrl: source.cellUrl, + sourceCellId: body.data.sourceCellId, + sourceCellIncarnation: body.data.sourceCellIncarnation, + audience: config.rehomeAudience, + identityToken: regionalRehomeIdentityToken, + fetch: regionalRehomeFetch + }) + return context.json(result) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/evacuate', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminAssignmentMoveSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const migration = await operations.assignments.startEvacuation( + { userId: body.data.userId, relayHostId: body.data.relayHostId }, + body.data.targetCellId + ) + return context.json({ v: 1, migration }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/migration-complete', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminMigrationCompleteSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + await operations.assignments.completeEvacuation( + { userId: body.data.userId, relayHostId: body.data.relayHostId }, + body.data.assignmentEpoch + ) + return context.json({ ok: true }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/migration-supersede-cell', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminRegisteredCellMigrationSupersedeSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const superseded = await operations.assignments.supersedeRegisteredCellEvacuations( + body.data.sourceCellId, + body.data.currentTargetCellId, + body.data.replacementTargetCellId, + body.data.limit + ) + return context.json({ v: 1, superseded }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/rebalance-dormant', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminAssignmentMoveSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const assignment = await operations.assignments.rebalanceDormant( + { userId: body.data.userId, relayHostId: body.data.relayHostId }, + body.data.targetCellId + ) + return context.json({ v: 1, assignment }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/admission-selector/apply', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminAdmissionSelectorApplySchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const result = await operations.assignments.applyCellAdmissionSelector(body.data) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/admission-selector/apply-staging-asia-proof', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminStagingAsiaProofAdmissionApplySchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const current = await operations.assignments.inspectCellAdmissionSelector() + const result = await operations.assignments.applyCellAdmissionSelector({ + attemptId: body.data.attemptId, + expectedGeneration: body.data.expectedGeneration, + membership: stagingAsiaProofMembership(current.selector.membership, body.data.state) + }) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/admission-selector/status', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminAdmissionSelectorStatusSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const result = await operations.assignments.inspectCellAdmissionSelector( + body.data.attemptId + ) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/admission-selector/add-migration-cells', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminAdmissionSelectorAddMigrationCellsSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const result = await operations.assignments.addMigrationCells({ + attemptId: body.data.attemptId, + expectedGeneration: body.data.expectedGeneration, + cells: body.data.cells.map((cell) => ({ + id: cell.cellId, + url: cell.cellUrl, + capacityRequests: cell.capacityRequests, + region: cell.region, + connectionHardCap: cell.connectionHardCap, + connectionUnobservedBound: cell.connectionUnobservedBound + })) + }) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-state', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellStateSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + await operations.assignments.setCellAdmissionState( + body.data.cellId, + 'state' in body.data + ? body.data.state + : body.data.enabled + ? 'general' + : 'existing-only' + ) + return context.json({ ok: true }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-adopt-legacy', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceLegacyAdoptionSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const expiresAt = await operations.assignments.adoptLegacyCellFence( + body.data.cellId, + body.data.cellIncarnation + ) + return context.json({ v: 1, cellId: body.data.cellId, expiresAt }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-commit-legacy-adoption', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceLegacyAdoptionCommitSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + await operations.assignments.commitLegacyCellFenceAdoption( + body.data.cellId, + body.data.cellIncarnation + ) + return context.json({ v: 1, cellId: body.data.cellId, committed: true }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-attest', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceAttestSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const evidence = cellFenceAttemptEvidence(body.data) + const result = await operations.assignments.attestCellFenceAttempt( + evidence, + body.data.gceOperation + ) + return context.json({ + v: 1, + cellId: body.data.cellId, + expiresAt: result.expiresAt, + attempt: result.attempt + }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-attempt-prepare', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceAttemptPrepareSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const evidence = cellFenceAttemptEvidence(body.data) + const attempt = await operations.assignments.prepareCellFenceAttempt(evidence) + return context.json({ v: 1, attempt }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-attempt-start', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceAttemptUpdateSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const evidence = cellFenceAttemptEvidence(body.data) + const result = await operations.assignments.startCellFenceApply( + evidence, + body.data.invocationId, + body.data.invocationRequestReason + ) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-attempt-plan', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFencePlanSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const evidence = cellFenceAttemptEvidence(body.data) + const attempt = await operations.assignments.bindCellFencePlanGeneration( + evidence, + body.data.planObjectGeneration + ) + return context.json({ v: 1, attempt }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-attempt-operation', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceOperationSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const evidence = cellFenceAttemptEvidence(body.data) + const result = await operations.assignments.recordCellFenceOperation( + evidence, + body.data.invocationId, + body.data.invocationRequestReason, + body.data.gceOperation + ) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-attempt-status', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceAttemptStatusSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const attempt = await operations.assignments.cellFenceAttempt(body.data.cellId) + return context.json({ v: 1, attempt }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-fence-attempt-abort', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellFenceAttemptAbortSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const evidence = cellFenceAttemptEvidence(body.data) + const attempt = await operations.assignments.abortCellFenceAttempt(evidence) + return context.json({ v: 1, attempt }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/drain-attempt-prepare', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminDrainAttemptPrepareSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const result = await operations.assignments.prepareCellDrainAttempt({ + attemptId: body.data.attemptId, + cellId: body.data.cellId, + cellIncarnation: body.data.cellIncarnation, + traceValue: body.data.traceValue, + plannedGraceMs: body.data.graceMs + }) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/drain-attempt-send', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminDrainAttemptMutationSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const attempt = await operations.assignments.beginCellDrainSend(body.data) + return context.json({ v: 1, attempt }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/drain-attempt-receipt', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminDrainAttemptReceiptSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const attempt = await operations.assignments.recordCellDrainApplicationReceipt( + body.data + ) + return context.json({ v: 1, attempt }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/drain-attempt-recover-forward', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminDrainAttemptRecoverSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const result = await operations.assignments.prepareCellDrainRecovery(body.data) + return context.json({ v: 1, ...result }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/cell-config', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellConfigSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + await operations.assignments.configureCell( + { + id: body.data.cellId, + url: body.data.cellUrl, + capacityRequests: body.data.capacityRequests, + connectionHardCap: body.data.connectionHardCap, + connectionUnobservedBound: body.data.connectionUnobservedBound + }, + 'state' in body.data ? body.data.state : body.data.enabled + ) + return context.json({ ok: true }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/evacuate-cell', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellEvacuateSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const started = await operations.assignments.startActiveCellEvacuations( + body.data.sourceCellId, + body.data.targetCellId, + body.data.limit + ) + return context.json({ v: 1, started }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/evacuation-capacity', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellEvacuationCapacitySchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const capacity = await operations.assignments.cellEvacuationCapacity( + body.data.sourceCellId, + body.data.targetCellId + ) + return context.json({ v: 1, ...capacity }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) + app.post('/v1/admin/evacuation-status', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellEvacuationStatusSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + if (body.data.completeReady && (await verifyReadOnlyAdminToken(bearer))) { + return context.json({ error: 'insufficient_permission' }, 403) + } + const status = await operations.assignments.cellEvacuationStatus( + body.data.sourceCellId, + body.data.targetCellId, + body.data.completeReady + ) + return context.json({ v: 1, ...status }) + }) + app.post('/v1/admin/cell-status', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminCellStatusSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + try { + const status = await operations.assignments.cellDeploymentStatus(body.data.cellId) + return context.json({ v: 1, status }) + } catch (error) { + return context.json({ error: operationError(error) }, 404) + } + }) + return app +} + +const AdminAssignmentMoveSchema = z + .object({ + v: z.literal(1), + userId: z.string().min(1).max(256), + relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/), + targetCellId: z.string().min(1).max(128) + }) + .strict() + +const RuntimeStatusSchema = z.object({ v: z.literal(1) }).strict() + +const RegionalRehomeSafetySchema = z + .object({ + observedAt: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + sqlFailures: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + reconnects: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + controlActivityRecoveryFailures: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER), + databasePoolWaiting: z.number().int().nonnegative().max(100), + databasePoolWaitersMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + databasePoolWaitMsMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + // Cells spread the full pool-pressure counts into the payload; rejecting + // the extra gauges 400'd every rehome-status heartbeat since 2026-08-15, + // so no cell ever recorded rehome protocol 1. + databasePoolTotal: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), + databasePoolIdle: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), + databasePoolOldestWaitMs: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER) + .optional() + }) + .strict() + +const CellHeartbeatSchema = z + .object({ + v: z.literal(1), + cellId: z.string().min(1).max(128), + cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin), + region: RelayRegionSchema.default(RELAY_DEFAULT_REGION), + cellIncarnation: z.string().uuid(), + startedAt: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + ready: z.boolean(), + observedRequests: z.number().int().nonnegative().max(RELAY_MAX_CELL_CAPACITY_REQUESTS), + totalConnections: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CAPACITY_REQUESTS) + .optional(), + inFlightConnections: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CAPACITY_REQUESTS) + .optional(), + reservedConnectionUnits: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CAPACITY_REQUESTS) + .optional(), + enforcedConnectionUnits: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CAPACITY_REQUESTS) + .optional(), + connectionInclusionWatermark: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER) + .optional(), + connectionHardCap: RelayCellConnectionHardCapSchema.optional(), + connectionUnobservedBound: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND) + .optional() + }) + .strict() + .superRefine((value, context) => { + const values = [ + value.totalConnections, + value.inFlightConnections, + value.reservedConnectionUnits, + value.enforcedConnectionUnits, + value.connectionHardCap, + value.connectionUnobservedBound + ] + if ( + values.some((candidate) => candidate !== undefined) && + values.some((candidate) => candidate === undefined) + ) { + context.addIssue({ + code: 'custom', + message: 'connection telemetry must be complete' + }) + } + if ( + value.enforcedConnectionUnits !== undefined && + value.totalConnections !== undefined && + value.inFlightConnections !== undefined && + value.reservedConnectionUnits !== undefined && + value.enforcedConnectionUnits !== + value.totalConnections + + value.inFlightConnections + + value.reservedConnectionUnits + ) { + context.addIssue({ + code: 'custom', + message: 'connection telemetry sum does not match' + }) + } + if ( + value.connectionHardCap !== undefined && + value.connectionUnobservedBound! > + relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound + ) { + context.addIssue({ + code: 'custom', + message: 'connection unobserved bound must leave ordinary admission capacity' + }) + } + }) + +const CellRegionalRehomeStatusSchema = z + .object({ + v: z.literal(1), + cellId: z.string().min(1).max(128), + cellIncarnation: z.string().uuid(), + regionalRehomeProtocol: z.number().int().min(0).max(1), + safety: RegionalRehomeSafetySchema + }) + .strict() + +const RegionalRehomeControlSchema = z.discriminatedUnion('action', [ + z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(), + z.object({ + v: z.literal(1), + action: z.literal('apply'), + expectedGeneration: z.number().int().nonnegative(), + enabled: z.boolean(), + notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + ratePerMinute: z.number().int().min(1).max(120), + preferenceMaxAgeMs: z + .number() + .int() + .min(60_000) + .max(30 * 24 * 60 * 60_000), + drainGraceMs: z.number().int().min(60_000).max(60 * 60_000), + confirmation: z.enum([ + 'ENABLE_REGIONAL_REHOMING', + 'DISABLE_REGIONAL_REHOMING' + ]) + }).strict() +]).superRefine((value, context) => { + if (value.action !== 'apply') return + const expected = value.enabled + ? 'ENABLE_REGIONAL_REHOMING' + : 'DISABLE_REGIONAL_REHOMING' + if (value.confirmation !== expected) { + context.addIssue({ code: 'custom', message: 'confirmation does not match state' }) + } +}) + +const RegionalRehomeTrustProbeSchema = z + .object({ + v: z.literal(1), + sourceCellId: z.string().min(1).max(128), + sourceCellIncarnation: z.string().uuid() + }) + .strict() + +const AdminMigrationCompleteSchema = z + .object({ + v: z.literal(1), + userId: z.string().min(1).max(256), + relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/), + assignmentEpoch: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) + }) + .strict() + +const AdminRegisteredCellMigrationSupersedeSchema = z + .object({ + v: z.literal(1), + sourceCellId: z.string().min(1).max(128), + currentTargetCellId: z.string().min(1).max(128), + replacementTargetCellId: z.string().min(1).max(128), + limit: z.number().int().min(1).max(100), + confirmation: z.literal('SUPERSEDE_REGISTERED_CELL_MIGRATIONS') + }) + .strict() + .refine( + (value) => + new Set([ + value.sourceCellId, + value.currentTargetCellId, + value.replacementTargetCellId + ]).size === 3 + ) + +const CellIdSchema = z.string().min(1).max(128) +const CellAdmissionStateSchema = z.enum(CELL_ADMISSION_STATES) +const AdmissionSelectorAttemptIdSchema = z.string().regex(/^[A-Za-z0-9_-]{8,128}$/) +const AdmissionSelectorMembershipSchema = z + .object({ + existingOnly: z.array(CellIdSchema).max(256), + migrationOnly: z.array(CellIdSchema).max(256), + general: z.array(CellIdSchema).max(256) + }) + .strict() + +const AdminAdmissionSelectorApplySchema = z + .object({ + v: z.literal(1), + attemptId: AdmissionSelectorAttemptIdSchema, + expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + expectedMembershipSha256: z.string().regex(/^[a-f0-9]{64}$/).optional(), + membership: AdmissionSelectorMembershipSchema + }) + .strict() + .refine( + (value) => value.expectedGeneration > 0 || value.expectedMembershipSha256 !== undefined + ) + +const AdminStagingAsiaProofAdmissionApplySchema = z + .object({ + v: z.literal(1), + attemptId: AdmissionSelectorAttemptIdSchema, + expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + state: z.enum(['general', 'migration-only']) + }) + .strict() + +const AdminAdmissionSelectorStatusSchema = z + .object({ v: z.literal(1), attemptId: AdmissionSelectorAttemptIdSchema.optional() }) + .strict() + +const AdminAdmissionSelectorAddMigrationCellsSchema = z + .object({ + v: z.literal(1), + attemptId: AdmissionSelectorAttemptIdSchema, + expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + cells: z + .array( + z + .object({ + cellId: CellIdSchema, + cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin), + capacityRequests: z + .number() + .int() + .positive() + .max(RELAY_MAX_CELL_CAPACITY_REQUESTS), + region: RelayRegionSchema.default(RELAY_DEFAULT_REGION), + connectionHardCap: RelayCellConnectionHardCapSchema, + connectionUnobservedBound: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND) + }) + .strict() + .superRefine((value, context) => { + if ( + value.connectionUnobservedBound > + relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound + ) { + context.addIssue({ + code: 'custom', + path: ['connectionUnobservedBound'], + message: 'connection unobserved bound must leave ordinary admission capacity' + }) + } + }) + ) + .min(1) + .max(128) + }) + .strict() + .refine( + ({ cells }) => + new Set(cells.map(({ cellId }) => cellId)).size === cells.length && + new Set(cells.map(({ cellUrl }) => cellUrl)).size === cells.length + ) + +const AdminCellStateSchema = z.union([ + z.object({ v: z.literal(1), cellId: CellIdSchema, enabled: z.boolean() }).strict(), + z.object({ v: z.literal(1), cellId: CellIdSchema, state: CellAdmissionStateSchema }).strict() +]) + +const CellConfigShape = { + v: z.literal(1), + cellId: CellIdSchema, + cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin), + capacityRequests: z.number().int().positive().max(RELAY_MAX_CELL_CAPACITY_REQUESTS), + connectionHardCap: RelayCellConnectionHardCapSchema.optional(), + connectionUnobservedBound: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND) + .optional() +} as const + +const AdminCellConfigSchema = z + .union([ + z + .object({ + ...CellConfigShape, + enabled: z.boolean() + }) + .strict(), + z + .object({ + ...CellConfigShape, + state: CellAdmissionStateSchema + }) + .strict() + ]) + .refine( + (value) => + (value.connectionHardCap === undefined) === + (value.connectionUnobservedBound === undefined), + { message: 'connection hard cap and unobserved bound must be configured together' } + ) + .refine( + (value) => + value.connectionHardCap === undefined || + value.connectionUnobservedBound! <= + relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound, + { message: 'connection unobserved bound must leave ordinary admission capacity' } + ) + +const CellPairShape = { + v: z.literal(1), + sourceCellId: z.string().min(1).max(128), + targetCellId: z.string().min(1).max(128) +} as const + +const AdminCellEvacuateSchema = z + .object({ ...CellPairShape, limit: z.number().int().min(1).max(100) }) + .strict() + .refine((value) => value.sourceCellId !== value.targetCellId) + +const AdminCellEvacuationCapacitySchema = z + .object(CellPairShape) + .strict() + .refine((value) => value.sourceCellId !== value.targetCellId) + +const AdminCellEvacuationStatusSchema = z + .object({ ...CellPairShape, completeReady: z.boolean().default(false) }) + .strict() + .refine((value) => value.sourceCellId !== value.targetCellId) + +const TerraformStateLineageSchema = z + .string() + .regex(/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i) + +const CellFenceAttemptBaseShape = { + attemptId: z.string().uuid(), + environment: z.enum(['staging', 'production']), + cellId: z.string().min(1).max(128), + cellIncarnation: z.string().uuid(), + migName: z.string().min(1).max(128), + instanceGroup: z.string().url().max(2048), + generationIdentity: z.string().url().max(2048), + fenceCommit: z.string().regex(/^[a-f0-9]{40}$/), + planSha256: z.string().regex(/^[a-f0-9]{64}$/), + planObjectName: z + .string() + .regex(/^terraform\/state\/relay-fence-plans\/(?:staging|production)\/[0-9a-f-]{36}\.tfplan$/), + varFileSha256: z.string().regex(/^[a-f0-9]{64}$/), + terraformStateLineage: TerraformStateLineageSchema, + terraformStateSerial: z.number().int().nonnegative().safe(), + terraformStateObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/), + terraformStateObjectSha256: z.string().regex(/^[a-f0-9]{64}$/), + requestReason: z + .string() + .regex(/^orca-relay-fence\/[0-9a-f]{8}-[0-9a-f-]{27}$/) +} as const +const CellFenceAttemptEvidenceShape = { + ...CellFenceAttemptBaseShape, + planObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/) +} as const + +const AdminCellFenceAttemptPrepareSchema = z + .object({ + v: z.literal(1), + ...CellFenceAttemptBaseShape, + confirmation: z.literal('PREPARE_TERRAFORM_CELL_FENCE') + }) + .strict() + +const AdminCellFencePlanSchema = z + .object({ + v: z.literal(1), + ...CellFenceAttemptEvidenceShape, + confirmation: z.literal('BIND_TERRAFORM_CELL_FENCE_PLAN') + }) + .strict() + +const AdminCellFenceAttemptUpdateSchema = z + .object({ + v: z.literal(1), + ...CellFenceAttemptEvidenceShape, + invocationId: z.string().uuid(), + invocationRequestReason: z.string().min(1).max(256), + confirmation: z.literal('START_TERRAFORM_CELL_FENCE') + }) + .strict() + +const AdminCellFenceOperationSchema = z + .object({ + v: z.literal(1), + ...CellFenceAttemptEvidenceShape, + invocationId: z.string().uuid(), + invocationRequestReason: z.string().min(1).max(256), + gceOperation: z.string().min(1).max(256), + confirmation: z.literal('RECORD_TERRAFORM_CELL_FENCE_OPERATION') + }) + .strict() + +const AdminCellFenceAttemptStatusSchema = z + .object({ v: z.literal(1), cellId: z.string().min(1).max(128) }) + .strict() + +const AdminCellFenceLegacyAdoptionSchema = z + .object({ + v: z.literal(1), + cellId: z.string().min(1).max(128), + cellIncarnation: z.string().uuid(), + confirmation: z.literal('ADOPT_LEGACY_TERRAFORM_CELL_FENCE') + }) + .strict() + +const AdminCellFenceLegacyAdoptionCommitSchema = z + .object({ + v: z.literal(1), + cellId: z.string().min(1).max(128), + cellIncarnation: z.string().uuid(), + confirmation: z.literal('COMMIT_LEGACY_TERRAFORM_CELL_FENCE_ADOPTION') + }) + .strict() + +const AdminCellFenceAttemptAbortSchema = z + .object({ + v: z.literal(1), + ...CellFenceAttemptBaseShape, + confirmation: z.literal('ABORT_UNSTARTED_TERRAFORM_CELL_FENCE') + }) + .strict() + +const AdminCellFenceAttestSchema = z + .object({ + v: z.literal(1), + ...CellFenceAttemptEvidenceShape, + gceOperation: z.string().min(1).max(256), + confirmation: z.literal('ATTEST_TERRAFORM_FENCED_CELL') + }) + .strict() + +const AdminDrainAttemptPrepareSchema = z + .object({ + v: z.literal(1), + attemptId: z.string().uuid(), + cellId: z.string().min(1).max(128), + cellIncarnation: z.string().uuid(), + traceValue: z.string().uuid(), + graceMs: z.literal(120_000), + confirmation: z.literal('PREPARE_LEGACY_DRAIN') + }) + .strict() + +const AdminDrainAttemptMutationSchema = z + .object({ + v: z.literal(1), + attemptId: z.string().uuid(), + cellId: z.string().min(1).max(128), + cellIncarnation: z.string().uuid() + }) + .strict() + +const AdminDrainAttemptReceiptSchema = AdminDrainAttemptMutationSchema.extend({ + traceValue: z.string().uuid(), + backendStatus: z.number().int().min(200).max(299), + backendInstance: z.string().min(1).max(256).optional() +}).strict() + +const AdminDrainAttemptRecoverSchema = z + .object({ + v: z.literal(1), + cellId: z.string().min(1).max(128), + cellIncarnation: z.string().uuid(), + confirmation: z.literal('RECOVER_LEGACY_DRAIN') + }) + .strict() + +const RegionalHostDrainSchema = z + .object({ + v: z.literal(1), + attemptId: z.string().uuid(), + userId: z.string().min(1).max(256), + relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/), + sourceCellId: z.string().min(1).max(128), + sourceCellIncarnation: z.string().uuid(), + sourceAssignmentEpoch: z.number().int().positive(), + graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) + }) + .strict() + +const AdminCellStatusSchema = z + .object({ v: z.literal(1), cellId: z.string().min(1).max(128) }) + .strict() + +function cellFenceAttemptEvidence( + value: CellFenceAttemptEvidence +): CellFenceAttemptEvidence { + return { + attemptId: value.attemptId, + environment: value.environment, + cellId: value.cellId, + cellIncarnation: value.cellIncarnation, + migName: value.migName, + instanceGroup: value.instanceGroup, + generationIdentity: value.generationIdentity, + fenceCommit: value.fenceCommit, + planSha256: value.planSha256, + planObjectName: value.planObjectName, + planObjectGeneration: value.planObjectGeneration, + varFileSha256: value.varFileSha256, + terraformStateLineage: value.terraformStateLineage, + terraformStateSerial: value.terraformStateSerial, + terraformStateObjectGeneration: value.terraformStateObjectGeneration, + terraformStateObjectSha256: value.terraformStateObjectSha256, + requestReason: value.requestReason + } +} + +function operationError(error: unknown): string { + return error instanceof Error ? error.message : 'operation_failed' +} + +export { relayHostLogDigest } + +// `suppressed` is present only on a window-closing line, and counts the rejections +// that line stands for beyond the one already logged when the window opened. +function logAssignmentRejection(input: { + route: 'assign' | 'assign-verify' | 'resolve' + lane: 'sticky' | 'placement' | 'none' + hinted: boolean + relayHostId: string + reason: string + suppressed?: number +}): void { + console.warn( + `[orca-relay] assignment rejected route=${input.route} lane=${input.lane}` + + ` hinted=${input.hinted} reason=${input.reason}` + + ` host=${relayHostLogDigest(input.relayHostId)}` + + (input.suppressed === undefined ? '' : ` suppressed=${input.suppressed}`) + ) +} + +function isRelayAssignmentCapacityError(error: unknown): boolean { + return ( + error instanceof Error && + ['relay_capacity_exhausted', 'relay_connection_headroom_exhausted'].includes( + error.message + ) + ) +} + +function isCanonicalRelayOrigin(value: string): boolean { + const url = new URL(value) + const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname) + return ( + url.origin === value && + url.pathname === '/' && + (url.protocol === 'https:' || (loopback && url.protocol === 'http:')) + ) +} + +function requestTooLarge(contentLength: string | undefined): boolean { + return Number(contentLength ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes +} diff --git a/cloud/apps/relay/src/assignment-cleanup-steps.test.ts b/cloud/apps/relay/src/assignment-cleanup-steps.test.ts new file mode 100644 index 00000000000..02c21f76ada --- /dev/null +++ b/cloud/apps/relay/src/assignment-cleanup-steps.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it, vi } from 'vitest' +import { + assignmentCleanupSteps, + runAssignmentCleanup, + type AssignmentCleanupStore +} from './assignment-cleanup-steps.js' + +function stubStore(overrides: Partial = {}) { + const calls: string[] = [] + const method = (name: string) => + vi.fn(async () => { + calls.push(name) + }) + const store: AssignmentCleanupStore = { + refreshRegionalRehomeLeases: method('refreshRegionalRehomeLeases'), + completeReadyEvacuations: method('completeReadyEvacuations'), + completeReadyRegionalRehomes: method('completeReadyRegionalRehomes'), + abortExpiredEvacuations: method('abortExpiredEvacuations'), + abortExpiredRegionalRehomes: method('abortExpiredRegionalRehomes'), + reapRegionalRehomeAttempts: method('reapRegionalRehomeAttempts'), + releaseExpiredActivityLeases: method('releaseExpiredActivityLeases'), + releaseExpiredActivity: method('releaseExpiredActivity'), + releaseExpiredRegionPreferences: method('releaseExpiredRegionPreferences'), + evacuateDeadCells: method('evacuateDeadCells'), + ...overrides + } + return { store, calls } +} + +describe('assignment cleanup steps', () => { + it('runs every later sweep when an early one fails, naming the step', async () => { + const { store, calls } = stubStore({ + completeReadyRegionalRehomes: vi.fn(async () => { + throw new Error('regional_rehome_assignment_mismatch') + }) + }) + const warn = vi.fn() + + await runAssignmentCleanup(store, warn) + + expect(calls).toEqual([ + 'refreshRegionalRehomeLeases', + 'completeReadyEvacuations', + 'abortExpiredEvacuations', + 'abortExpiredRegionalRehomes', + 'reapRegionalRehomeAttempts', + 'releaseExpiredActivityLeases', + 'releaseExpiredActivity', + 'releaseExpiredRegionPreferences', + 'evacuateDeadCells' + ]) + expect(warn).toHaveBeenCalledTimes(1) + expect(String(warn.mock.calls[0]![0])).toContain( + '[orca-relay] assignment cleanup failed: complete-ready-regional-rehomes' + ) + }) + + it('covers all ten sweeps exactly once per run', async () => { + const { store, calls } = stubStore() + + await runAssignmentCleanup(store) + + expect(calls).toHaveLength(10) + expect(new Set(calls).size).toBe(10) + expect(assignmentCleanupSteps(store)).toHaveLength(10) + }) +}) diff --git a/cloud/apps/relay/src/assignment-cleanup-steps.ts b/cloud/apps/relay/src/assignment-cleanup-steps.ts new file mode 100644 index 00000000000..0b6547d2ae4 --- /dev/null +++ b/cloud/apps/relay/src/assignment-cleanup-steps.ts @@ -0,0 +1,52 @@ +import { runRelayBackgroundOperation } from './relay-background-operation.js' + +// The ten periodic assignment sweeps the director runs every 30s. Each step +// re-derives its state from the database and is idempotent, so they carry no +// intra-tick ordering dependency — which is what makes per-step isolation +// sound: one failing sweep costs one tick of itself, never the other nine. +// (A single poisoned rehome row once silenced the whole chained form +// fleet-wide.) Sweep failures are logged, never fed into the rehome worker's +// dispatch-failure budget: a sweep exception is not a dispatch failure and +// must not durably disable regional rehoming. +export type AssignmentCleanupStore = { + refreshRegionalRehomeLeases(): Promise + completeReadyEvacuations(): Promise + completeReadyRegionalRehomes(): Promise + abortExpiredEvacuations(): Promise + abortExpiredRegionalRehomes(): Promise + reapRegionalRehomeAttempts(): Promise + releaseExpiredActivityLeases(): Promise + releaseExpiredActivity(): Promise + releaseExpiredRegionPreferences(): Promise + evacuateDeadCells(): Promise +} + +export function assignmentCleanupSteps( + assignments: AssignmentCleanupStore +): ReadonlyArray Promise]> { + return [ + ['refresh-regional-rehome-leases', () => assignments.refreshRegionalRehomeLeases()], + ['complete-ready-evacuations', () => assignments.completeReadyEvacuations()], + ['complete-ready-regional-rehomes', () => assignments.completeReadyRegionalRehomes()], + ['abort-expired-evacuations', () => assignments.abortExpiredEvacuations()], + ['abort-expired-regional-rehomes', () => assignments.abortExpiredRegionalRehomes()], + ['reap-regional-rehome-attempts', () => assignments.reapRegionalRehomeAttempts()], + ['release-expired-activity-leases', () => assignments.releaseExpiredActivityLeases()], + ['release-expired-activity', () => assignments.releaseExpiredActivity()], + ['release-expired-region-preferences', () => assignments.releaseExpiredRegionPreferences()], + ['evacuate-dead-cells', () => assignments.evacuateDeadCells()] + ] +} + +export async function runAssignmentCleanup( + assignments: AssignmentCleanupStore, + warn?: (message: string) => void +): Promise { + for (const [step, operation] of assignmentCleanupSteps(assignments)) { + await runRelayBackgroundOperation( + operation, + `[orca-relay] assignment cleanup failed: ${step}`, + warn + ) + } +} diff --git a/cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts b/cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts new file mode 100644 index 00000000000..6ac9521c3d6 --- /dev/null +++ b/cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts @@ -0,0 +1,342 @@ +import pg from 'pg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { ASSIGNMENT_CONNECTION_HEADROOM_QUERY } from './assignment-connection-headroom-query.js' +import { RelayAssignmentStore } from './assignment-store.js' +import { + openRelayDatabase, + type RelayDatabase +} from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const headroomIndexName = 'relay_control_connection_reservation_headroom' +const cell = { + id: 'connection-headroom-postgres', + url: 'https://connection-headroom-postgres.example.com', + capacityRequests: 1_000, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 +} + +describePostgres('PostgreSQL assignment connection headroom', () => { + const databases: RelayDatabase[] = [] + + beforeAll(async () => { + databases.push( + await openRelayDatabase({ databaseUrl, dataDir: '' }), + await openRelayDatabase({ databaseUrl, dataDir: '' }), + await openRelayDatabase({ databaseUrl, dataDir: '' }) + ) + }) + + afterAll(async () => { + const database = databases[0] + if (database) { + await database.query( + `DELETE FROM relay_control_connection_reservations + WHERE user_id LIKE 'connection-headroom-postgres-%'` + ) + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id LIKE 'connection-headroom-postgres-%'` + ) + await database.query( + `DELETE FROM relay_assignments + WHERE user_id LIKE 'connection-headroom-postgres-%'` + ) + await database.query( + `DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, + [cell.id] + ) + await database.query( + `DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, + [cell.id] + ) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id]) + } + for (const connection of databases) await connection.close() + }) + + it('commits only one parallel assignment at the admission boundary', async () => { + const stores = databases.map( + (database) => + new RelayAssignmentStore(database, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + ) + await databases[0]!.query( + `DELETE FROM relay_control_connection_reservations + WHERE user_id LIKE 'connection-headroom-postgres-%'` + ) + await databases[0]!.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id LIKE 'connection-headroom-postgres-%'` + ) + await databases[0]!.query( + `DELETE FROM relay_assignments + WHERE user_id LIKE 'connection-headroom-postgres-%'` + ) + await stores[0]!.reconcileCells([cell], false) + await stores[0]!.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + for (const suffix of ['1', '2']) { + await databases[0]!.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + `connection-headroom-postgres-${suffix}`, + `headroomhost000${suffix}`, + cell.id, + 1, + 90_100, + 100, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ) + } + + const results = await Promise.allSettled([ + stores[1]!.assign({ + userId: 'connection-headroom-postgres-1', + relayHostId: 'headroomhost0001' + }), + stores[2]!.assign({ + userId: 'connection-headroom-postgres-2', + relayHostId: 'headroomhost0002' + }) + ]) + expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1) + expect(results.filter(({ status }) => status === 'rejected')).toHaveLength(1) + + const assignments = await databases[0]!.query( + `SELECT COALESCE(SUM(reserved_controls), 0) AS count FROM relay_assignments + WHERE user_id LIKE 'connection-headroom-postgres-%'` + ) + const pending = await databases[0]!.query( + `SELECT COUNT(*) AS count FROM relay_assignment_activity_leases + WHERE user_id LIKE 'connection-headroom-postgres-%' + AND activity_kind = 'control' + AND activity_id LIKE 'control-pending:%'` + ) + const reservations = await databases[0]!.query( + `SELECT COUNT(*) AS count FROM relay_control_connection_reservations + WHERE user_id LIKE 'connection-headroom-postgres-%' + AND state <> 'released'` + ) + expect(Number(assignments[0]!.count)).toBe(1) + expect(Number(pending[0]!.count)).toBe(1) + expect(Number(reservations[0]!.count)).toBe(1) + }, 15_000) + + it('uses the composite headroom index when released history dominates', async () => { + await databases[0]!.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, created_at, timeout_at, updated_at) + SELECT 'headroom-plan-' || value, 'headroom-plan-' || value, + 'connection-headroom-postgres-plan', 'plan-host-' || value, + 1, ?, 'released', 1, 2, 1 + FROM generate_series(1, 50000) AS value`, + [cell.id] + ) + await databases[0]!.query(`ANALYZE relay_control_connection_reservations`) + + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + let reservationIndexPlan: Record | undefined + try { + const plan = await client.query( + `EXPLAIN (FORMAT JSON) ${ASSIGNMENT_CONNECTION_HEADROOM_QUERY}` + ) + reservationIndexPlan = findReservationHeadroomIndexPlan( + plan.rows[0]?.['QUERY PLAN'] + ) + } finally { + await client.end() + } + + expect(reservationIndexPlan).toBeDefined() + expect(String(reservationIndexPlan?.['Index Cond'])).toContain('cell_id') + expect(String(reservationIndexPlan?.['Index Cond'])).toContain('state = ANY') + expect(reservationIndexPlan?.['Filter']).toBeUndefined() + }) + + it('deduplicates expired debt after a fresh PostgreSQL snapshot', async () => { + const now = 200 + const store = new RelayAssignmentStore(databases[0]!, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + const identity = { + userId: 'connection-headroom-postgres-debt', + relayHostId: 'headroomdebt0001' + } + await databases[0]!.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + VALUES + (?, ?, ?, ?, 1, ?, 'late-arrival-debt', NULL, NULL, 100, 150, NULL, NULL, 150), + (?, ?, ?, ?, 1, ?, 'late-arrival-debt', NULL, NULL, 101, 150, NULL, NULL, 150)`, + [ + 'postgres-debt-1', + 'postgres-debt-1', + identity.userId, + identity.relayHostId, + cell.id, + 'postgres-debt-2', + 'postgres-debt-2', + identity.userId, + identity.relayHostId, + cell.id + ] + ) + + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionInclusionWatermark: 10, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + + expect( + await databases[0]!.query( + `SELECT state + FROM relay_control_connection_reservations + WHERE user_id = ? + ORDER BY created_at`, + [identity.userId] + ) + ).toEqual([ + { state: 'late-arrival-debt' }, + { state: 'released' } + ]) + }) + + it('releases an aborted older epoch after a fresh PostgreSQL snapshot', async () => { + const now = 300 + const store = new RelayAssignmentStore(databases[0]!, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + const identity = { + userId: 'connection-headroom-postgres-aborted', + relayHostId: 'headroomabort001' + } + await databases[0]!.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, 3, ?, ?, 0, 0, 0, 0, 0, 0)`, + [identity.userId, identity.relayHostId, cell.id, now, now] + ) + await databases[0]!.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, 1, 2, 0, 1, ?, ?, NULL, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + 'source', + cell.id, + now - 2, + now - 3, + now - 1, + now - 4, + now - 1 + ] + ) + await databases[0]!.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + VALUES (?, ?, ?, ?, 2, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`, + [ + 'postgres-aborted-reservation', + 'postgres-aborted-reservation', + identity.userId, + identity.relayHostId, + cell.id, + now - 4, + now - 2, + now - 1 + ] + ) + + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionInclusionWatermark: 20, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + + expect( + await databases[0]!.query( + `SELECT state FROM relay_control_connection_reservations + WHERE reservation_id = ?`, + ['postgres-aborted-reservation'] + ) + ).toEqual([{ state: 'released' }]) + }) +}) + +function findReservationHeadroomIndexPlan( + value: unknown +): Record | undefined { + if (value === null || typeof value !== 'object') return undefined + const record = value as Record + if (!Array.isArray(value) && record['Index Name'] === headroomIndexName) return record + for (const child of Object.values(value)) { + const match = findReservationHeadroomIndexPlan(child) + if (match) return match + } + return undefined +} diff --git a/cloud/apps/relay/src/assignment-connection-headroom-query.ts b/cloud/apps/relay/src/assignment-connection-headroom-query.ts new file mode 100644 index 00000000000..c0237aecf26 --- /dev/null +++ b/cloud/apps/relay/src/assignment-connection-headroom-query.ts @@ -0,0 +1,15 @@ +export const ASSIGNMENT_CONNECTION_HEADROOM_QUERY = + `SELECT limits.cell_id, limits.hard_cap, limits.unobserved_bound, + connection_snapshot.enforced_connection_units, + connection_snapshot.snapshot_at AS last_heartbeat_at, + connection_snapshot.cell_incarnation AS connection_incarnation, + current_runtime.cell_incarnation AS current_incarnation, + (SELECT COUNT(*) FROM relay_control_connection_reservations reservation + WHERE reservation.cell_id = limits.cell_id + AND reservation.state IN + ('reserved', 'late-arrival-debt', 'claimed')) AS outstanding_reservations + FROM relay_cell_connection_limits limits + LEFT JOIN relay_cell_connection_snapshots connection_snapshot + ON connection_snapshot.cell_id = limits.cell_id + LEFT JOIN relay_cell_runtime current_runtime + ON current_runtime.cell_id = limits.cell_id` diff --git a/cloud/apps/relay/src/assignment-connection-headroom.test.ts b/cloud/apps/relay/src/assignment-connection-headroom.test.ts new file mode 100644 index 00000000000..19f5d46aa33 --- /dev/null +++ b/cloud/apps/relay/src/assignment-connection-headroom.test.ts @@ -0,0 +1,1007 @@ +import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract' +import { afterEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayCellConfig } from './config.js' +import { + openInMemoryRelayDatabase, + type RelayDatabase +} from './database.js' + +const LIMITED_CELL: RelayCellConfig = { + id: 'limited', + url: 'https://limited.example.com', + capacityRequests: 1_000, + connectionHardCap: 600, + connectionUnobservedBound: 50 +} + +const databases: RelayDatabase[] = [] + +afterEach(async () => { + for (const database of databases.splice(0)) await database.close() +}) + +async function setup( + connectionUnits: number, + now: () => number = () => 100 +): Promise<{ database: RelayDatabase; store: RelayAssignmentStore }> { + const database = await openInMemoryRelayDatabase() + databases.push(database) + const store = new RelayAssignmentStore(database, now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells([LIMITED_CELL], true) + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: connectionUnits, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: connectionUnits, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + return { database, store } +} + +async function setupHeadroomReassignment(): Promise<{ + database: RelayDatabase + store: RelayAssignmentStore + source: RelayCellConfig + target: RelayCellConfig +}> { + const database = await openInMemoryRelayDatabase() + databases.push(database) + const store = new RelayAssignmentStore(database, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + const source = { + ...LIMITED_CELL, + id: 'saturated-source', + url: 'https://saturated-source.example.com' + } + const target = { + ...LIMITED_CELL, + id: 'available-target', + url: 'https://available-target.example.com' + } + await store.reconcileCells([source, target], true) + await store.recordCellHeartbeat({ + cellId: source.id, + cellUrl: source.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 450, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 450, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + await store.recordCellHeartbeat({ + cellId: target.id, + cellUrl: target.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + return { database, store, source, target } +} + +describe('relay assignment connection headroom', () => { + it('requires exact, internally consistent telemetry from limited cells', async () => { + const database = await openInMemoryRelayDatabase() + databases.push(database) + const store = new RelayAssignmentStore(database, () => 100) + await store.reconcileCells([LIMITED_CELL], true) + const heartbeat = { + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0 + } + + await expect(store.recordCellHeartbeat(heartbeat)).rejects.toThrow( + 'cell_connection_telemetry_mismatch' + ) + await expect( + store.recordCellHeartbeat({ + ...heartbeat, + totalConnections: 550, + inFlightConnections: 2, + reservedConnectionUnits: 3, + enforcedConnectionUnits: 554, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + ).rejects.toThrow('cell_connection_telemetry_mismatch') + await expect( + store.recordCellHeartbeat({ + ...heartbeat, + totalConnections: 601, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 601, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + ).resolves.toBeUndefined() + }) + + it('reports the limited-cell capacity contract and telemetry components', async () => { + const { store } = await setup(0) + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 120, + inFlightConnections: 2, + reservedConnectionUnits: 3, + enforcedConnectionUnits: 125, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + + expect((await store.cellDeploymentStatus(LIMITED_CELL.id)).connectionCapacity).toEqual({ + hardCap: 600, + controlRebindReserve: 100, + ordinaryConnectionLimit: 500, + unobservedBound: 50, + normalAdmissionPause: 450, + observedConnections: 120, + inFlightConnections: 2, + reservedConnectionUnits: 3, + enforcedConnectionUnits: 125, + pendingControlReservations: 0, + heartbeatFresh: true + }) + }) + + it('fails placement closed while a cell changes connection limits', async () => { + const { store } = await setup(449) + const expanded = { + ...LIMITED_CELL, + connectionHardCap: 1_000 as const + } + + await store.reconcileCells([expanded], true) + await expect( + store.assign({ userId: 'transition-user', relayHostId: 'host000000000099' }) + ).rejects.toThrow('relay_capacity_exhausted') + await expect(store.cellDeploymentStatus(expanded.id)).resolves.toMatchObject({ + connectionCapacity: { hardCap: 1_000, heartbeatFresh: false } + }) + + await store.recordCellHeartbeat({ + cellId: expanded.id, + cellUrl: expanded.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 200, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionHardCap: 1_000, + connectionUnobservedBound: 50 + }) + await expect(store.cellDeploymentStatus(expanded.id)).resolves.toMatchObject({ + connectionCapacity: { + hardCap: 1_000, + ordinaryConnectionLimit: 900, + normalAdmissionPause: 850, + heartbeatFresh: true + } + }) + }) + + it('fails placement closed while the admin path changes connection limits', async () => { + const { store } = await setup(449) + const expanded = { + ...LIMITED_CELL, + connectionHardCap: 1_000 as const + } + + await store.configureCell(expanded, 'general') + await expect( + store.assign({ userId: 'admin-transition', relayHostId: 'host000000000098' }) + ).rejects.toThrow('relay_capacity_exhausted') + await expect(store.cellDeploymentStatus(expanded.id)).resolves.toMatchObject({ + connectionCapacity: { hardCap: 1_000, heartbeatFresh: false } + }) + + await store.recordCellHeartbeat({ + cellId: expanded.id, + cellUrl: expanded.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 200, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionHardCap: 1_000, + connectionUnobservedBound: 50 + }) + await expect( + store.assign({ userId: 'admin-restored', relayHostId: 'host000000000097' }) + ).resolves.toMatchObject({ cellId: expanded.id }) + }) + + it('admits exactly one reservation at the admission boundary', async () => { + const { database, store } = await setup(449) + const results = await Promise.allSettled([ + store.assign({ userId: 'user-1', relayHostId: 'host000000000001' }), + store.assign({ userId: 'user-2', relayHostId: 'host000000000002' }) + ]) + + expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1) + expect( + results.filter(({ status }) => status === 'rejected').map((result) => + result.status === 'rejected' ? result.reason : null + ) + ).toEqual([expect.objectContaining({ message: 'relay_capacity_exhausted' })]) + expect( + await database.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE activity_kind = 'control' AND activity_id LIKE 'control-pending:%'` + ) + ).toHaveLength(1) + }) + + it('rejects at the boundary before mutating durable assignment state', async () => { + const { database, store } = await setup(450) + + await expect( + store.assign({ userId: 'user-1', relayHostId: 'host000000000001' }) + ).rejects.toThrow('relay_capacity_exhausted') + expect(await database.query(`SELECT * FROM relay_assignments`)).toEqual([]) + expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([]) + }) + + it('charges every active reservation state at the admission boundary', async () => { + const { database, store } = await setup(447) + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, created_at, timeout_at, updated_at) + VALUES + ('active-reserved', 'active-reserved', 'state-user', 'state-host-1', + 1, ?, 'reserved', 1, 2, 1), + ('active-debt', 'active-debt', 'state-user', 'state-host-2', + 1, ?, 'late-arrival-debt', 1, 2, 1), + ('active-claimed', 'active-claimed', 'state-user', 'state-host-3', + 1, ?, 'claimed', 1, 2, 1)`, + [LIMITED_CELL.id, LIMITED_CELL.id, LIMITED_CELL.id] + ) + + await expect( + store.assign({ userId: 'blocked-user', relayHostId: 'blockedhost00001' }) + ).rejects.toThrow('relay_capacity_exhausted') + expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([]) + }) + + it('does not charge released reservation history', async () => { + const { database, store } = await setup(449) + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, created_at, timeout_at, updated_at) + VALUES ('released-history', 'released-history', 'state-user', 'state-host', + 1, ?, 'released', 1, 2, 1)`, + [LIMITED_CELL.id] + ) + + await expect( + store.assign({ userId: 'admitted-user', relayHostId: 'admittedhost0001' }) + ).resolves.toMatchObject({ cellId: LIMITED_CELL.id }) + }) + + it('rejects sticky control restoration before mutating assignment state', async () => { + const { database, store } = await setup(450) + const identity = { userId: 'sticky-user', relayHostId: 'stickyhost000001' } + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + LIMITED_CELL.id, + 1, + 10_000, + 100, + 0, + 0, + 1, + 0, + 0, + 0 + ] + ) + + await expect(store.assign(identity)).rejects.toThrow( + 'relay_connection_headroom_exhausted' + ) + expect( + await database.query( + `SELECT cell_id, assignment_epoch, reserved_controls, reserved_invites + FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { + cell_id: LIMITED_CELL.id, + assignment_epoch: 1, + reserved_controls: 0, + reserved_invites: 1 + } + ]) + expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([]) + }) + + it('moves a zero-activity assignment off a general cell without connection headroom', async () => { + const { database, store, source, target } = await setupHeadroomReassignment() + const identity = { userId: 'movable-user', relayHostId: 'movablehost000001' } + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0)`, + [identity.userId, identity.relayHostId, source.id, 7, 10_000, 100] + ) + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`, + [ + 'superseded-reservation', + 'superseded-reservation', + identity.userId, + identity.relayHostId, + 7, + source.id, + 50, + 99, + 100 + ] + ) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: target.id, + assignmentEpoch: 8 + }) + expect( + await database.query( + `SELECT cell_id, assignment_epoch, reserved_controls + FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ cell_id: target.id, assignment_epoch: 8, reserved_controls: 1 }]) + expect( + await database.query( + `SELECT cell_id, activity_kind FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ cell_id: target.id, activity_kind: 'control' }]) + expect( + await database.query( + `SELECT cell_id, assignment_epoch, state + FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ? + ORDER BY assignment_epoch`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { cell_id: source.id, assignment_epoch: 7, state: 'released' }, + { cell_id: target.id, assignment_epoch: 8, state: 'reserved' } + ]) + }) + + it('keeps non-control activity pinned when its cell has no connection headroom', async () => { + const { database, store, source } = await setupHeadroomReassignment() + const identity = { userId: 'pinned-user', relayHostId: 'pinnedhost0000001' } + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, 0, 0, 1, 0, 0, 0)`, + [identity.userId, identity.relayHostId, source.id, 7, 10_000, 100] + ) + + await expect(store.assign(identity)).rejects.toThrow( + 'relay_connection_headroom_exhausted' + ) + expect(await store.resolve(identity)).toMatchObject({ + cellId: source.id, + assignmentEpoch: 7 + }) + }) + + it.each(['existing-only', 'migration-only'] as const)( + 'keeps a zero-activity assignment pinned on a %s cell without connection headroom', + async (admission) => { + const { database, store, source } = await setupHeadroomReassignment() + const identity = { + userId: `pinned-${admission}-user`, + relayHostId: `pinned${admission.replace('-', '')}` + } + await store.setCellAdmissionState(source.id, admission) + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0)`, + [identity.userId, identity.relayHostId, source.id, 7, 10_000, 100] + ) + + await expect(store.assign(identity)).rejects.toThrow( + 'relay_connection_headroom_exhausted' + ) + expect(await store.resolve(identity)).toMatchObject({ + cellId: source.id, + assignmentEpoch: 7 + }) + expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([]) + } + ) + + it('renames a pending reservation exactly once on control activation', async () => { + const { database, store } = await setup(449) + const identity = { userId: 'user-1', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + + await store.activateControl(identity, { + cellId: LIMITED_CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.activateControl(identity, { + cellId: LIMITED_CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + + expect( + await database.query( + `SELECT activity_id, request_units FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ activity_id: 'control:limited:1', request_units: 1 }]) + }) + + it('keeps expired pending reservations charged as late-arrival debt', async () => { + let now = 100 + const { database, store } = await setup(449, () => now) + await store.assign({ userId: 'user-1', relayHostId: 'host000000000001' }) + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + await store.releaseExpiredActivityLeases() + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + + await expect( + store.assign({ userId: 'user-2', relayHostId: 'host000000000002' }) + ).rejects.toThrow('relay_capacity_exhausted') + expect( + await database.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE activity_kind = 'control' AND activity_id LIKE 'control-pending:%'` + ) + ).toHaveLength(0) + expect( + await database.query( + `SELECT state FROM relay_control_connection_reservations` + ) + ).toEqual([{ state: 'late-arrival-debt' }]) + }) + + it('reconciles late-arrival debt only after a covering absolute snapshot', async () => { + let now = 100 + const { database, store } = await setup(449, () => now) + const identity = { userId: 'late-user', relayHostId: 'latehost00000001' } + const assignment = await store.assign(identity) + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + await store.releaseExpiredActivityLeases() + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionInclusionWatermark: 4, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + + await store.activateControl(identity, { + cellId: LIMITED_CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1, + connectionInclusionWatermark: 5 + }) + expect( + await database.query( + `SELECT state, inclusion_watermark FROM relay_control_connection_reservations` + ) + ).toEqual([{ state: 'claimed', inclusion_watermark: 5 }]) + + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 450, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 450, + connectionInclusionWatermark: 5, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + expect( + await database.query( + `SELECT state, released_at FROM relay_control_connection_reservations` + ) + ).toEqual([{ state: 'released', released_at: now }]) + await expect( + store.assign({ userId: 'blocked-user', relayHostId: 'blockedhost00001' }) + ).rejects.toThrow('relay_capacity_exhausted') + }) + + it('rejects duplicate and out-of-order absolute snapshots', async () => { + const { database, store } = await setup(0) + const heartbeat = { + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 10, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 10, + connectionInclusionWatermark: 10, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 + } + await store.recordCellHeartbeat(heartbeat) + await expect(store.recordCellHeartbeat(heartbeat)).rejects.toThrow( + 'stale_connection_snapshot' + ) + await expect( + store.recordCellHeartbeat({ + ...heartbeat, + totalConnections: 9, + enforcedConnectionUnits: 9, + connectionInclusionWatermark: 9 + }) + ).rejects.toThrow('stale_connection_snapshot') + expect( + await database.query( + `SELECT inclusion_watermark, enforced_connection_units + FROM relay_cell_connection_snapshots` + ) + ).toEqual([{ inclusion_watermark: 10, enforced_connection_units: 10 }]) + }) + + it('keeps crash retries bound to one reservation claim', async () => { + let now = 100 + const { database, store } = await setup(448, () => now) + const identity = { userId: 'retry-user', relayHostId: 'retryhost0000001' } + const assignment = await store.assign(identity) + await new RelayAssignmentStore(database, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }).assign(identity) + expect( + await database.query( + `SELECT state FROM relay_control_connection_reservations` + ) + ).toEqual([{ state: 'reserved' }]) + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + await store.releaseExpiredActivityLeases() + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 448, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 448, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + await store.assign(identity) + const restarted = new RelayAssignmentStore(database, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + + const activation = { + cellId: LIMITED_CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1, + connectionInclusionWatermark: 5 + } + await restarted.activateControl(identity, activation) + await restarted.activateControl(identity, activation) + + expect( + await database.query( + `SELECT state, claim_activity_id + FROM relay_control_connection_reservations + ORDER BY created_at ASC, reservation_id ASC` + ) + ).toEqual([{ state: 'claimed', claim_activity_id: 'control:limited:1' }]) + }) + + it('releases only redundant expired debt after a fresh absolute snapshot', async () => { + let now = 100 + const { database, store } = await setup(449, () => now) + const identity = { userId: 'debt-user', relayHostId: 'debthost000000001' } + const assignment = await store.assign(identity) + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + await store.releaseExpiredActivityLeases() + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`, + [ + 'duplicate-reservation', + 'duplicate-reservation', + identity.userId, + identity.relayHostId, + assignment.assignmentEpoch, + LIMITED_CELL.id, + 101, + now - 1, + now + ] + ) + + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 449, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 449, + connectionInclusionWatermark: 5, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + + expect( + await database.query( + `SELECT state, COUNT(*) AS count + FROM relay_control_connection_reservations + GROUP BY state ORDER BY state` + ) + ).toEqual([ + { state: 'late-arrival-debt', count: 1 }, + { state: 'released', count: 1 } + ]) + }) + + it('releases expired debt for a snapshot-proven aborted older epoch', async () => { + const now = 1_000 + const { database, store } = await setup(0, () => now) + const identity = { userId: 'aborted-user', relayHostId: 'abortedhost00001' } + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0)`, + [identity.userId, identity.relayHostId, LIMITED_CELL.id, 3, now, now] + ) + await database.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, 1, 2, 0, 1, ?, ?, NULL, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + 'source', + LIMITED_CELL.id, + now - 2, + now - 3, + now - 1, + now - 4, + now - 1 + ] + ) + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + VALUES (?, ?, ?, ?, 2, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`, + [ + 'aborted-reservation', + 'aborted-reservation', + identity.userId, + identity.relayHostId, + LIMITED_CELL.id, + now - 4, + now - 2, + now - 1 + ] + ) + + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionInclusionWatermark: 5, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + + expect( + await database.query( + `SELECT state FROM relay_control_connection_reservations + WHERE reservation_id = ?`, + ['aborted-reservation'] + ) + ).toEqual([{ state: 'released' }]) + }) + + it('fails a limited cell closed on stale telemetry while leaving legacy cells unchanged', async () => { + let now = 100 + const { store } = await setup(0, () => now) + now += 45_001 + await expect( + store.assign({ userId: 'user-1', relayHostId: 'host000000000001' }) + ).rejects.toThrow('relay_capacity_exhausted') + + const legacyDatabase = await openInMemoryRelayDatabase() + databases.push(legacyDatabase) + const legacyStore = new RelayAssignmentStore(legacyDatabase, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + const legacy = { + id: 'legacy', + url: 'https://legacy.example.com', + capacityRequests: 10 + } + await legacyStore.reconcileCells([legacy], true) + await legacyStore.recordCellHeartbeat({ + cellId: legacy.id, + cellUrl: legacy.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: now, + ready: true, + observedRequests: 0 + }) + await expect( + legacyStore.assign({ userId: 'user-2', relayHostId: 'host000000000002' }) + ).resolves.toMatchObject({ cellId: 'legacy' }) + }) + + it('does not create connection debt for mixed-version uncapped cells', async () => { + let now = 100 + const legacy = { + id: 'legacy', + url: 'https://legacy.example.com', + capacityRequests: 10 + } + const { database, store } = await setup(0, () => now) + await store.reconcileCells([legacy], true) + await store.recordCellHeartbeat({ + cellId: legacy.id, + cellUrl: legacy.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 50, + ready: true, + observedRequests: 0 + }) + + await store.assign({ userId: 'legacy-user', relayHostId: 'legacyhost000001' }) + expect( + await database.query(`SELECT * FROM relay_control_connection_reservations`) + ).toEqual([]) + expect(await store.releaseExpiredActivityLeases()).toBe(0) + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + expect(await store.releaseExpiredActivityLeases()).toBe(1) + expect( + await database.query(`SELECT * FROM relay_control_connection_reservations`) + ).toEqual([]) + }) + + it('rejects dormant rebalance before changing the assignment epoch', async () => { + const now = ASSIGNMENT_LIMITS.dormantTtlMs + 100 + const database = await openInMemoryRelayDatabase() + databases.push(database) + const store = new RelayAssignmentStore(database, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + const source = { + id: 'source', + url: 'https://source.example.com', + capacityRequests: 10 + } + await store.reconcileCells([source, LIMITED_CELL], true) + await store.recordCellHeartbeat({ + cellId: source.id, + cellUrl: source.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: now - 50, + ready: true, + observedRequests: 0 + }) + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: now - 50, + ready: true, + observedRequests: 0, + totalConnections: 450, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 450, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + const identity = { userId: 'dormant-user', relayHostId: 'dormanthost00001' } + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + source.id, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ) + + await expect(store.rebalanceDormant(identity, LIMITED_CELL.id)).rejects.toThrow( + 'relay_connection_headroom_exhausted' + ) + expect(await store.resolve(identity)).toMatchObject({ + cellId: source.id, + assignmentEpoch: 7 + }) + expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([]) + }) + + it('rejects an evacuation target at its pause before migration mutation', async () => { + const database = await openInMemoryRelayDatabase() + databases.push(database) + const store = new RelayAssignmentStore(database, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + const source = { + id: 'source', + url: 'https://source.example.com', + capacityRequests: 10 + } + await store.reconcileCells( + [source, { ...LIMITED_CELL, initiallyEnabled: false }], + true + ) + await store.recordCellHeartbeat({ + cellId: source.id, + cellUrl: source.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 50, + ready: true, + observedRequests: 0 + }) + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 450, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 450, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + const identity = { userId: 'user-1', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + expect(assignment.cellId).toBe(source.id) + await store.setCellEnabled(LIMITED_CELL.id, true) + + await expect( + store.startEvacuation(identity, LIMITED_CELL.id) + ).rejects.toThrow('relay_connection_headroom_exhausted') + expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) + expect(await store.resolve(identity)).toMatchObject({ + cellId: source.id, + assignmentEpoch: assignment.assignmentEpoch + }) + }) +}) diff --git a/cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts b/cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts new file mode 100644 index 00000000000..10193b78cc6 --- /dev/null +++ b/cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts @@ -0,0 +1,130 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const cell = { + id: 'control-supersession-postgres', + url: 'https://control-supersession-postgres.example.com', + capacityRequests: 1_000, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 +} +const identity = { + userId: 'control-supersession-postgres-user', + relayHostId: 'supersedehost001' +} + +describePostgres('PostgreSQL control supersession', () => { + const databases: RelayDatabase[] = [] + + beforeAll(async () => { + databases.push( + await openRelayDatabase({ databaseUrl, dataDir: '' }), + await openRelayDatabase({ databaseUrl, dataDir: '' }) + ) + }) + + afterAll(async () => { + const database = databases[0] + if (database) { + await database.query( + `DELETE FROM relay_control_connection_reservations WHERE user_id = ?`, + [identity.userId] + ) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, + [identity.userId] + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId]) + await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id]) + } + for (const connection of databases) await connection.close() + }) + + it('serializes parallel generations into one durable control', async () => { + const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100)) + await stores[0]!.reconcileCells([cell]) + await stores[0]!.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionInclusionWatermark: 1, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + const assignment = await stores[0]!.assign(identity) + + await Promise.all([ + stores[0]!.activateControl(identity, { + cellId: cell.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1, + connectionInclusionWatermark: 10 + }), + stores[1]!.activateControl(identity, { + cellId: cell.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 2, + connectionInclusionWatermark: 11 + }) + ]) + await databases[0]!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, 'control', ?, 1, 90100, 100), + (?, ?, ?, 'control', ?, 1, 90100, 100)`, + [ + identity.userId, + identity.relayHostId, + `control:${cell.id}:100`, + cell.id, + identity.userId, + identity.relayHostId, + `control:${cell.id}:101`, + cell.id + ] + ) + await stores[0]!.activateControl(identity, { + cellId: cell.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 102, + connectionInclusionWatermark: 12 + }) + + const controls = await databases[0]!.query( + `SELECT COUNT(*) AS count FROM relay_assignment_activity_leases + WHERE user_id = ? AND activity_kind = 'control'`, + [identity.userId] + ) + expect(Number(controls[0]!.count)).toBe(1) + const assignments = await databases[0]!.query( + `SELECT reserved_controls FROM relay_assignments WHERE user_id = ?`, + [identity.userId] + ) + expect(Number(assignments[0]!.reserved_controls)).toBe(1) + const cells = await databases[0]!.query( + `SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`, + [cell.id] + ) + expect(Number(cells[0]!.reserved_requests)).toBe(1) + const claims = await databases[0]!.query( + `SELECT claim_activity_id FROM relay_control_connection_reservations + WHERE user_id = ? AND state = 'claimed'`, + [identity.userId] + ) + expect(claims).toEqual([{ claim_activity_id: `control:${cell.id}:102` }]) + }, 15_000) +}) diff --git a/cloud/apps/relay/src/assignment-deployment-status-postgres.test.ts b/cloud/apps/relay/src/assignment-deployment-status-postgres.test.ts new file mode 100644 index 00000000000..9e8e3b85bd9 --- /dev/null +++ b/cloud/apps/relay/src/assignment-deployment-status-postgres.test.ts @@ -0,0 +1,87 @@ +import pg from 'pg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const schema = 'relay_deployment_status_test' +const cell = { + id: 'deployment-status-postgres', + url: 'https://deployment-status-postgres.example.com', + capacityRequests: 20 +} + +describePostgres('PostgreSQL deployment status', () => { + let database: RelayDatabase | undefined + + beforeAll(async () => { + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + await client.query(`CREATE SCHEMA ${schema}`) + } finally { + await client.end() + } + const url = new URL(databaseUrl!) + url.searchParams.set('options', `-c search_path=${schema}`) + database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' }) + }) + + afterAll(async () => { + await database?.close() + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + } finally { + await client.end() + } + }) + + it('classifies pending controls and restart blockers in one PostgreSQL query', async () => { + const store = new RelayAssignmentStore(database!, () => 100) + await store.reconcileCells([cell]) + await store.assign({ userId: 'status-user', relayHostId: 'a1b2c3d4e5f6' }) + + expect(await store.cellDeploymentStatus(cell.id)).toMatchObject({ + activityLeases: 1, + activityRequestUnits: 1, + reservedRequests: 1, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: 0 + }) + + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES ('blocking-user', 'b1c2d3e4f5a6', 'invite:test', 'invite', ?, 1, 90100, 100), + ('malformed-user', 'c1d2e3f4a5b6', 'control-pending:bad', 'control', ?, 2, 90100, 100)`, + [cell.id, cell.id] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = reserved_requests + 3 WHERE cell_id = ?`, + [cell.id] + ) + + expect(await store.cellDeploymentStatus(cell.id)).toMatchObject({ + activityLeases: 3, + activityRequestUnits: 4, + reservedRequests: 4, + restartBlockingActivityLeases: 2, + restartBlockingActivityRequestUnits: 3, + restartBlockingReservedRequests: 3 + }) + + await database!.query( + `UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id = ?`, + [cell.id] + ) + expect(await store.cellDeploymentStatus(cell.id)).toMatchObject({ + restartBlockingReservedRequests: -1 + }) + }) +}) diff --git a/cloud/apps/relay/src/assignment-identity-queue.test.ts b/cloud/apps/relay/src/assignment-identity-queue.test.ts new file mode 100644 index 00000000000..930f0b692df --- /dev/null +++ b/cloud/apps/relay/src/assignment-identity-queue.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' +import { AssignmentIdentityQueue } from './assignment-identity-queue.js' + +function deferred(): { + promise: Promise + resolve: () => void +} { + let resolve!: () => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} + +describe('AssignmentIdentityQueue', () => { + it('serializes operations for the same assignment', async () => { + const queue = new AssignmentIdentityQueue() + const firstStarted = deferred() + const firstRelease = deferred() + const started: string[] = [] + const identity = { userId: 'user-1', relayHostId: 'host-1' } + + const first = queue.run(identity, async () => { + started.push('first') + firstStarted.resolve() + await firstRelease.promise + }) + const second = queue.run(identity, async () => { + started.push('second') + }) + + await firstStarted.promise + expect(started).toEqual(['first']) + firstRelease.resolve() + await Promise.all([first, second]) + expect(started).toEqual(['first', 'second']) + }) + + it('allows different assignments to run concurrently', async () => { + const queue = new AssignmentIdentityQueue() + const release = deferred() + const started: string[] = [] + + const first = queue.run({ userId: 'user-1', relayHostId: 'host-1' }, async () => { + started.push('first') + await release.promise + }) + const second = queue.run({ userId: 'user-2', relayHostId: 'host-1' }, async () => { + started.push('second') + }) + + await second + expect(started).toEqual(['first', 'second']) + release.resolve() + await first + }) + + it('continues after a rejected operation', async () => { + const queue = new AssignmentIdentityQueue() + const identity = { userId: 'user-1', relayHostId: 'host-1' } + + const failed = queue.run(identity, async () => { + throw new Error('failed') + }) + const recovered = queue.run(identity, async () => 'recovered') + + await expect(failed).rejects.toThrow('failed') + await expect(recovered).resolves.toBe('recovered') + }) +}) diff --git a/cloud/apps/relay/src/assignment-identity-queue.ts b/cloud/apps/relay/src/assignment-identity-queue.ts new file mode 100644 index 00000000000..f39d9c0a03c --- /dev/null +++ b/cloud/apps/relay/src/assignment-identity-queue.ts @@ -0,0 +1,24 @@ +export type AssignmentIdentity = { + userId: string + relayHostId: string +} + +export class AssignmentIdentityQueue { + private readonly tails = new Map>() + + async run(identity: AssignmentIdentity, operation: () => Promise): Promise { + const key = JSON.stringify([identity.userId, identity.relayHostId]) + const previous = this.tails.get(key) ?? Promise.resolve() + const result = previous.catch(() => undefined).then(operation) + const tail = result.then( + () => undefined, + () => undefined + ) + this.tails.set(key, tail) + try { + return await result + } finally { + if (this.tails.get(key) === tail) this.tails.delete(key) + } + } +} diff --git a/cloud/apps/relay/src/assignment-inventory-snapshot.test.ts b/cloud/apps/relay/src/assignment-inventory-snapshot.test.ts new file mode 100644 index 00000000000..afc35f7e601 --- /dev/null +++ b/cloud/apps/relay/src/assignment-inventory-snapshot.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest' +import { + formatAssignmentInventorySnapshot, + readAssignmentInventorySnapshot +} from './assignment-inventory-snapshot.js' +import { openInMemoryRelayDatabase } from './database.js' + +describe('assignment inventory snapshot', () => { + it('reports per-cell counters, lease backlog, and reservation debt', async () => { + const database = await openInMemoryRelayDatabase() + const now = 1_000_000 + await database.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES ('cell-a', 'https://a.example.test', 1, 4000, 3999, 5, ?, ?)`, + [now, now] + ) + await database.query( + `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at) + VALUES ('cell-a', 'general', ?)`, + [now] + ) + await database.query( + `INSERT INTO relay_cell_runtime + (cell_id, cell_url, cell_incarnation, started_at, ready, observed_requests, + last_heartbeat_at, updated_at) + VALUES ('cell-a', 'https://a.example.test', 'inc-1', ?, 1, 5, ?, ?)`, + [now - 60_000, now - 10_000, now] + ) + await database.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units, + expires_at, updated_at) + VALUES + ('user-1', 'host-1', 'control:1', 'control', 'cell-a', 1, ?, ?), + ('user-1', 'host-2', 'control:1', 'control', 'cell-a', 3, ?, ?)`, + [now - 1, now, now + 90_000, now] + ) + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch, + cell_id, state, created_at, timeout_at, updated_at) + VALUES + ('r1', 'k1', 'user-1', 'host-1', 1, 'cell-a', 'late-arrival-debt', ?, ?, ?), + ('r2', 'k2', 'user-1', 'host-2', 1, 'cell-a', 'reserved', ?, ?, ?), + ('r3', 'k3', 'user-1', 'host-3', 1, 'cell-a', 'claimed', ?, ?, ?), + ('r4', 'k4', 'user-1', 'host-4', 1, 'cell-a', 'released', ?, ?, ?)`, + [now, now, now, now, now, now, now, now, now, now, now, now] + ) + + const snapshot = await readAssignmentInventorySnapshot(database, now) + + expect(snapshot.cells).toEqual([ + { + cellId: 'cell-a', + region: 'us-central1', + admissionState: 'general', + enabled: true, + capacityRequests: 4000, + reservedRequests: 3999, + runtimeReady: true, + heartbeatAgeMs: 10_000 + } + ]) + expect(snapshot.activityLeases).toEqual({ total: 2, expired: 1, requestUnits: 4 }) + expect(snapshot.connectionReservations).toEqual({ outstanding: 3, lateArrivalDebt: 1 }) + expect(snapshot.regionalRehomes).toEqual({ + active: 0, + awaitingReceipt: 0, + targetRegistered: 0, + completedLast24Hours: 0, + abortedLast24Hours: 0, + oldestActiveAgeMs: null + }) + + const lines = formatAssignmentInventorySnapshot(snapshot) + expect(lines).toHaveLength(3) + expect(lines[0]).toContain('cellId=cell-a') + expect(lines[0]).toContain('reserved=3999') + expect(lines[1]).toContain('expiredLeases=1') + expect(lines[1]).toContain('lateArrivalDebt=1') + expect(lines[2]).toContain('active=0') + }) + + it('reports cells missing runtime and admission rows without failing', async () => { + const database = await openInMemoryRelayDatabase() + await database.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES ('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 0)` + ) + + const snapshot = await readAssignmentInventorySnapshot(database, 5_000) + + expect(snapshot.cells[0]).toMatchObject({ + cellId: 'cell-b', + admissionState: 'unset', + enabled: false, + runtimeReady: null, + heartbeatAgeMs: null + }) + expect(snapshot.activityLeases).toEqual({ total: 0, expired: 0, requestUnits: 0 }) + expect(snapshot.regionalRehomes.active).toBe(0) + }) +}) diff --git a/cloud/apps/relay/src/assignment-inventory-snapshot.ts b/cloud/apps/relay/src/assignment-inventory-snapshot.ts new file mode 100644 index 00000000000..0675bd49b94 --- /dev/null +++ b/cloud/apps/relay/src/assignment-inventory-snapshot.ts @@ -0,0 +1,170 @@ +import type { RelayDatabase, SqlRow } from './database.js' + +export type CellInventorySnapshotRow = { + cellId: string + region: string + admissionState: string + enabled: boolean + capacityRequests: number + reservedRequests: number + runtimeReady: boolean | null + heartbeatAgeMs: number | null +} + +export type AssignmentInventorySnapshot = { + cells: CellInventorySnapshotRow[] + activityLeases: { total: number; expired: number; requestUnits: number } + connectionReservations: { outstanding: number; lateArrivalDebt: number } + regionalRehomes: { + active: number + awaitingReceipt: number + targetRegistered: number + completedLast24Hours: number + abortedLast24Hours: number + oldestActiveAgeMs: number | null + } +} + +// Plain SELECTs only: this snapshot must never take the cell-inventory lock, +// or observability itself would add to the assign-path lock contention. +export async function readAssignmentInventorySnapshot( + database: RelayDatabase, + now: number +): Promise { + const cellRows = await database.query( + `SELECT cell.cell_id, cell.enabled, cell.capacity_requests, cell.reserved_requests, + admission.admission_state, region.region, + runtime.ready AS runtime_ready, runtime.last_heartbeat_at AS runtime_heartbeat_at + FROM relay_cells cell + LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id + LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id + LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id + ORDER BY cell.cell_id ASC` + ) + const leaseRow = ( + await database.query( + `SELECT COUNT(*) AS total, + COALESCE(SUM(CASE WHEN expires_at <= ? THEN 1 ELSE 0 END), 0) AS expired, + COALESCE(SUM(request_units), 0) AS request_units + FROM relay_assignment_activity_leases`, + [now] + ) + )[0] + const reservationRow = ( + await database.query( + // relay_cells is append-only in production; joining it lets the composite + // index skip released history without hiding any production cell's debt. + `SELECT COUNT(reservation.reservation_id) AS outstanding, + COALESCE(SUM(CASE WHEN reservation.state = 'late-arrival-debt' + THEN 1 ELSE 0 END), 0) AS late_arrival_debt + FROM relay_cells cell + LEFT JOIN relay_control_connection_reservations reservation + ON reservation.cell_id = cell.cell_id + AND reservation.state IN ('reserved', 'late-arrival-debt', 'claimed')` + ) + )[0] + const regionalRehomeRow = ( + await database.query( + `SELECT + COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + THEN 1 ELSE 0 END), 0) AS active, + COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + AND attempt.drain_receipt_at IS NULL + THEN 1 ELSE 0 END), 0) AS awaiting_receipt, + COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + AND migration.target_registered_at IS NOT NULL + THEN 1 ELSE 0 END), 0) AS target_registered, + COALESCE(SUM(CASE WHEN attempt.completed_at >= ? THEN 1 ELSE 0 END), 0) + AS completed_last_24_hours, + COALESCE(SUM(CASE WHEN attempt.aborted_at >= ? THEN 1 ELSE 0 END), 0) + AS aborted_last_24_hours, + MIN(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + THEN attempt.created_at END) AS oldest_active_at + FROM relay_region_rehome_attempts attempt + LEFT JOIN relay_assignment_migrations migration + ON migration.user_id = attempt.user_id + AND migration.relay_host_id = attempt.relay_host_id + AND migration.assignment_epoch = attempt.assignment_epoch`, + [now - 24 * 60 * 60_000, now - 24 * 60 * 60_000] + ) + )[0] + const oldestActiveAt = optionalInteger(regionalRehomeRow, 'oldest_active_at') + return { + cells: cellRows.map((row) => ({ + cellId: asText(row, 'cell_id'), + region: optionalText(row, 'region') ?? 'us-central1', + admissionState: optionalText(row, 'admission_state') ?? 'unset', + enabled: asInteger(row, 'enabled') === 1, + capacityRequests: asInteger(row, 'capacity_requests'), + reservedRequests: asInteger(row, 'reserved_requests'), + runtimeReady: row['runtime_ready'] == null ? null : asInteger(row, 'runtime_ready') === 1, + heartbeatAgeMs: + row['runtime_heartbeat_at'] == null ? null : now - asInteger(row, 'runtime_heartbeat_at') + })), + activityLeases: { + total: asInteger(leaseRow, 'total'), + expired: asInteger(leaseRow, 'expired'), + requestUnits: asInteger(leaseRow, 'request_units') + }, + connectionReservations: { + outstanding: asInteger(reservationRow, 'outstanding'), + lateArrivalDebt: asInteger(reservationRow, 'late_arrival_debt') + }, + regionalRehomes: { + active: asInteger(regionalRehomeRow, 'active'), + awaitingReceipt: asInteger(regionalRehomeRow, 'awaiting_receipt'), + targetRegistered: asInteger(regionalRehomeRow, 'target_registered'), + completedLast24Hours: asInteger(regionalRehomeRow, 'completed_last_24_hours'), + abortedLast24Hours: asInteger(regionalRehomeRow, 'aborted_last_24_hours'), + oldestActiveAgeMs: oldestActiveAt === null ? null : now - oldestActiveAt + } + } +} + +export function formatAssignmentInventorySnapshot( + snapshot: AssignmentInventorySnapshot +): string[] { + const lines = snapshot.cells.map( + (cell) => + `[orca-relay] cell inventory cellId=${cell.cellId}` + + ` region=${cell.region}` + + ` admission=${cell.admissionState} enabled=${cell.enabled}` + + ` capacity=${cell.capacityRequests} reserved=${cell.reservedRequests}` + + ` ready=${cell.runtimeReady ?? 'none'}` + + ` heartbeatAgeMs=${cell.heartbeatAgeMs ?? 'none'}` + ) + lines.push( + `[orca-relay] lease inventory leases=${snapshot.activityLeases.total}` + + ` expiredLeases=${snapshot.activityLeases.expired}` + + ` leaseRequestUnits=${snapshot.activityLeases.requestUnits}` + + ` outstandingReservations=${snapshot.connectionReservations.outstanding}` + + ` lateArrivalDebt=${snapshot.connectionReservations.lateArrivalDebt}` + ) + lines.push( + `[orca-relay] regional rehome inventory active=${snapshot.regionalRehomes.active}` + + ` awaitingReceipt=${snapshot.regionalRehomes.awaitingReceipt}` + + ` targetRegistered=${snapshot.regionalRehomes.targetRegistered}` + + ` completedLast24Hours=${snapshot.regionalRehomes.completedLast24Hours}` + + ` abortedLast24Hours=${snapshot.regionalRehomes.abortedLast24Hours}` + + ` oldestActiveAgeMs=${snapshot.regionalRehomes.oldestActiveAgeMs ?? 'none'}` + ) + return lines +} + +function asText(row: SqlRow | undefined, column: string): string { + return String(row?.[column] ?? '') +} + +function optionalText(row: SqlRow | undefined, column: string): string | null { + const value = row?.[column] + return value == null ? null : String(value) +} + +function asInteger(row: SqlRow | undefined, column: string): number { + return Number(row?.[column] ?? 0) +} + +function optionalInteger(row: SqlRow | undefined, column: string): number | null { + const value = row?.[column] + return value == null ? null : Number(value) +} diff --git a/cloud/apps/relay/src/assignment-rejection-log-window.test.ts b/cloud/apps/relay/src/assignment-rejection-log-window.test.ts new file mode 100644 index 00000000000..8ef078eff86 --- /dev/null +++ b/cloud/apps/relay/src/assignment-rejection-log-window.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' +import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js' + +describe('assignment rejection log window', () => { + it('emits once per key per window and closes it with its own suppressed count', () => { + const timers = manualTimers() + const closed: { key: string; suppressed: number; sample: string }[] = [] + const window = new AssignmentRejectionLogWindow({ + windowMs: 10_000, + schedule: timers.schedule, + onWindowClosed: (input) => closed.push(input) + }) + const key = 'assign:placement:host-rate-limited' + + expect(window.admit(key, 'host-a')).toBe(true) + expect(window.admit(key, 'host-b')).toBe(false) + expect(window.admit(key, 'host-c')).toBe(false) + expect(closed).toEqual([]) + + timers.runPending() + expect(closed).toEqual([{ key, suppressed: 2, sample: 'host-c' }]) + + // The next window starts clean instead of inheriting the closed window's count. + expect(window.admit(key, 'host-d')).toBe(true) + timers.runPending() + expect(closed).toHaveLength(1) + }) + + it('reports the final count for a key that goes quiet', () => { + const timers = manualTimers() + const closed: { key: string; suppressed: number }[] = [] + const window = new AssignmentRejectionLogWindow({ + windowMs: 10_000, + schedule: timers.schedule, + onWindowClosed: ({ key, suppressed }) => closed.push({ key, suppressed }) + }) + + window.admit('assign:sticky:host-rate-limited', 'host-a') + window.admit('assign:sticky:host-rate-limited', 'host-a') + window.admit('assign:sticky:host-rate-limited', 'host-a') + // No further rejection ever arrives for this key. + timers.runPending() + + expect(closed).toEqual([{ key: 'assign:sticky:host-rate-limited', suppressed: 2 }]) + }) + + it('keeps distinct keys on independent windows', () => { + const timers = manualTimers() + const window = new AssignmentRejectionLogWindow({ + windowMs: 10_000, + schedule: timers.schedule, + onWindowClosed: () => undefined + }) + + expect(window.admit('assign:placement:host-rate-limited', 'host-a')).toBe(true) + expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(true) + expect(window.admit('assign:sticky:host-rate-limited', 'host-a')).toBe(true) + expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(false) + }) + + it('bounds the tracked keys and reports what an evicted window suppressed', () => { + const timers = manualTimers() + const closed: { key: string; suppressed: number }[] = [] + const window = new AssignmentRejectionLogWindow({ + windowMs: 10_000, + schedule: timers.schedule, + onWindowClosed: ({ key, suppressed }) => closed.push({ key, suppressed }) + }) + + window.admit('key-0', 'host-a') + window.admit('key-0', 'host-b') + for (let index = 1; index < 200; index++) window.admit(`key-${index}`, 'host-a') + + expect(closed[0]).toEqual({ key: 'key-0', suppressed: 1 }) + // Evicted keys emit again rather than staying silently suppressed. + expect(window.admit('key-0', 'host-a')).toBe(true) + expect(window.admit('key-199', 'host-a')).toBe(false) + }) +}) + +function manualTimers(): { + schedule: (callback: () => void) => () => void + runPending: () => void +} { + const pending = new Map void>() + let nextId = 0 + return { + schedule: (callback) => { + const id = nextId++ + pending.set(id, callback) + return () => pending.delete(id) + }, + runPending: () => { + for (const [id, callback] of [...pending]) { + pending.delete(id) + callback() + } + } + } +} diff --git a/cloud/apps/relay/src/assignment-rejection-log-window.ts b/cloud/apps/relay/src/assignment-rejection-log-window.ts new file mode 100644 index 00000000000..21ad4e72af9 --- /dev/null +++ b/cloud/apps/relay/src/assignment-rejection-log-window.ts @@ -0,0 +1,60 @@ +type CancelWindow = () => void + +// Admission rejections run at ~17/s per director instance, so the log summarizes +// instead of streaming: one line when a key's window opens, then a closing line +// carrying whatever that same window suppressed. The closing line is driven by the +// window's own timer, so a key that falls quiet still reports its final count +// instead of waiting for a rejection that may never arrive. +const MAX_TRACKED_KEYS = 64 + +type RejectionWindow = { + suppressed: number + sample: TSample + cancel: CancelWindow +} + +export class AssignmentRejectionLogWindow { + private readonly windows = new Map>() + + constructor( + private readonly options: { + windowMs: number + onWindowClosed: (input: { key: string; suppressed: number; sample: TSample }) => void + schedule?: (callback: () => void, delayMs: number) => CancelWindow + } + ) {} + + // Returns true when the caller should log this rejection immediately. + admit(key: string, sample: TSample): boolean { + const open = this.windows.get(key) + if (open) { + open.suppressed++ + // Keep the most recent host/reason so the closing line names a real rejection. + open.sample = sample + return false + } + const schedule = this.options.schedule ?? defaultSchedule + const window: RejectionWindow = { suppressed: 0, sample, cancel: () => undefined } + this.windows.set(key, window) + if (this.windows.size > MAX_TRACKED_KEYS) { + this.close(this.windows.keys().next().value!) + } + window.cancel = schedule(() => this.close(key), this.options.windowMs) + return true + } + + private close(key: string): void { + const window = this.windows.get(key) + if (!window) return + this.windows.delete(key) + window.cancel() + if (window.suppressed === 0) return + this.options.onWindowClosed({ key, suppressed: window.suppressed, sample: window.sample }) + } +} + +function defaultSchedule(callback: () => void, delayMs: number): CancelWindow { + const timer = setTimeout(callback, delayMs) + timer.unref?.() + return () => clearTimeout(timer) +} diff --git a/cloud/apps/relay/src/assignment-rejection-logging.test.ts b/cloud/apps/relay/src/assignment-rejection-logging.test.ts new file mode 100644 index 00000000000..b00d791fce9 --- /dev/null +++ b/cloud/apps/relay/src/assignment-rejection-logging.test.ts @@ -0,0 +1,244 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RelayAssignment } from './assignment-store.js' +import type { RelayConfig } from './config.js' + +const fakes = vi.hoisted(() => ({ + verifyRelayToken: vi.fn(async (token: string) => ({ + sub: 'user-1', + relayHostId: token + })) +})) + +vi.mock('./relay-token-verifier.js', () => ({ + createRelayTokenVerifier: () => fakes.verifyRelayToken, + readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null +})) + +import { createRelayApp, relayHostLogDigest } from './app.js' + +describe('assignment rejection logging', () => { + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + it('logs the store reason and host digest when assign() rejects on capacity', async () => { + const host = 'hhhhhhhhhhhhhhhh' + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign: vi.fn(async () => { + throw new Error('relay_capacity_exhausted') + }), + resolve: vi.fn(async () => assignment('cell-r', host)) + } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const hinted = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true })) + + expect(hinted.status).toBe(503) + expect(await hinted.json()).toEqual({ error: 'relay_capacity_exhausted' }) + const line = warn.mock.calls.map((call) => String(call[0])).find((entry) => + entry.includes('assignment rejected') + ) + expect(line).toContain('route=assign') + expect(line).toContain('lane=sticky') + expect(line).toContain('hinted=true') + expect(line).toContain('reason=relay_capacity_exhausted') + expect(line).toContain(`host=${relayHostLogDigest(host)}`) + expect(line).not.toContain(host) + }) + + it('logs an unhinted placement rejection without the raw host id', async () => { + const host = 'gggggggggggggggg' + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign: vi.fn(async () => { + throw new Error('relay_connection_headroom_exhausted') + }), + resolve: vi.fn(async () => null) + } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const response = await app.request('/v1/assign', assignmentRequest(host)) + + expect(response.status).toBe(503) + const line = warn.mock.calls.map((call) => String(call[0])).find((entry) => + entry.includes('assignment rejected') + ) + expect(line).toContain('route=assign') + expect(line).toContain('lane=placement') + expect(line).toContain('hinted=false') + expect(line).toContain('reason=relay_connection_headroom_exhausted') + expect(line).not.toContain(host) + }) + + it('names the throttled host once per window and closes it with the suppressed count', async () => { + vi.useFakeTimers() + const host = 'mmmmmmmmmmmmmmmm' + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign: vi.fn(async () => assignment('cell-r', host)), + resolve: vi.fn(async () => null) + } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200) + expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503) + expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503) + + const throttled = (): string[] => + warn.mock.calls + .map((call) => String(call[0])) + .filter((entry) => entry.includes('reason=host-rate-limited')) + expect(throttled()).toHaveLength(1) + expect(throttled()[0]).toContain('route=assign') + expect(throttled()[0]).toContain('lane=placement') + expect(throttled()[0]).toContain(`host=${relayHostLogDigest(host)}`) + expect(throttled()[0]).not.toContain('suppressed=') + expect(throttled()[0]).not.toContain(host) + + await vi.advanceTimersByTimeAsync(10_000) + expect(throttled()).toHaveLength(2) + expect(throttled()[1]).toContain('suppressed=1') + expect(throttled()[1]).toContain(`host=${relayHostLogDigest(host)}`) + }) + + it('stays silent for successful assignments', async () => { + const host = 'ssssssssssssssss' + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign: vi.fn(async () => assignment('cell-r', host)), + resolve: vi.fn(async () => null) + } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200) + expect( + warn.mock.calls.map((call) => String(call[0])).filter((entry) => + entry.includes('assignment rejected') + ) + ).toEqual([]) + }) +}) + +function assignmentRequest( + relayHostId: string, + extra: Record = {} +): RequestInit & { headers: Record } { + return { + method: 'POST', + headers: { + authorization: `Bearer ${relayHostId}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, relayHostId, ...extra }) + } +} + +function assignment(cellId: string, relayHostId: string): RelayAssignment { + return { + userId: 'user-1', + relayHostId, + cellId, + cellUrl: `https://${cellId}.relay.example.test`, + assignmentEpoch: 1, + leaseExpiresAt: Date.now() + 300_000 + } +} + +function config(overrides: Partial = {}): RelayConfig { + return { + port: 8080, + publicUrl: 'https://relay.example.test', + cellUrl: 'https://relay.example.test', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'), + role: 'director', + cellId: 'director', + cells: [], + adminAudience: 'https://relay.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + runtimeServiceAccount: 'runtime@example.test', + adminJwksUrl: 'https://auth.example.test/jwks', + databasePoolMax: 3, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './data', + ...overrides + } +} + +describe('assignment grant logging', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('logs sticky grants with cell id and host digest only', async () => { + const host = 'gggggggggggggggg' + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign: vi.fn(async () => assignment('cell-r', host)), + resolve: vi.fn(async () => assignment('cell-r', host)) + } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true })) + + expect(response.status).toBe(200) + const line = warn.mock.calls.map((call) => String(call[0])).find((entry) => + entry.includes('assignment granted') + ) + expect(line).toContain('lane=sticky') + expect(line).toContain('cell=cell-r') + expect(line).toContain(`host=${relayHostLogDigest(host)}`) + expect(line).not.toContain(host) + }) + + it('does not log unhinted placement grants', async () => { + const host = 'pppppppppppppppp' + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign: vi.fn(async () => assignment('cell-r', host)), + resolve: vi.fn(async () => null) + } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200) + expect( + warn.mock.calls.map((call) => String(call[0])).filter((entry) => + entry.includes('assignment granted') + ) + ).toEqual([]) + }) +}) diff --git a/cloud/apps/relay/src/assignment-store-lock-order.test.ts b/cloud/apps/relay/src/assignment-store-lock-order.test.ts new file mode 100644 index 00000000000..61baedfb1e2 --- /dev/null +++ b/cloud/apps/relay/src/assignment-store-lock-order.test.ts @@ -0,0 +1,486 @@ +import { describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayDatabase, RelayLockOptions, SqlRow } from './database.js' + +const identity = { userId: 'user-a', relayHostId: 'host000000000001' } +const activityId = 'splice:connection-1' + +class LockOrderDatabase implements RelayDatabase { + readonly lockedTables: string[] = [] + + constructor( + private readonly cleanupCandidate: boolean, + private readonly currentLeaseExpiresAt: number, + private readonly activityLeasePresent = true + ) {} + + async query(sql: string): Promise { + if (sql.includes('SELECT user_id, relay_host_id, activity_id')) { + return this.cleanupCandidate + ? [ + { + user_id: identity.userId, + relay_host_id: identity.relayHostId, + activity_id: activityId + } + ] + : [] + } + if ( + sql.includes('UPDATE relay_cells SET reserved_requests') && + sql.includes('RETURNING cell_id') + ) { + this.lockedTables.push('cell') + return [{ cell_id: 'cell-a' }] + } + return [{ changes: 1 }] + } + + async queryLocked(sql: string): Promise { + if (sql.includes('FROM relay_assignments ')) { + this.lockedTables.push('assignment') + return [{ cell_id: 'cell-a', assignment_epoch: 1 }] + } + if (sql.includes('FROM relay_assignment_activity_leases')) { + this.lockedTables.push('activity') + if (!this.activityLeasePresent) return [] + return [ + { + user_id: identity.userId, + relay_host_id: identity.relayHostId, + activity_id: activityId, + activity_kind: 'splice', + cell_id: 'cell-a', + request_units: 2, + expires_at: this.currentLeaseExpiresAt + } + ] + } + if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') { + this.lockedTables.push('cell-inventory') + return [{ cell_id: 'cell-a', reserved_requests: 3, capacity_requests: 10 }] + } + if (sql.includes('FROM relay_cells')) { + this.lockedTables.push('cell') + return [{ reserved_requests: 3, capacity_requests: 10 }] + } + return [] + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await operation(this) + } + + async close(): Promise {} +} + +class ReassignmentLockOrderDatabase implements RelayDatabase { + readonly locks: string[] = [] + + async query(sql: string): Promise { + if (sql.includes('SELECT cell_id, region FROM relay_cell_regions')) { + return ['cell-a', 'cell-b'].map((cell_id) => ({ cell_id, region: 'us-central1' })) + } + if (sql.includes('SELECT region FROM relay_cell_regions')) { + return [{ region: 'us-central1' }] + } + if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) { + return [cellRow('cell-b', 1)] + } + if (sql.includes('JOIN relay_cell_runtime') && sql.includes('cell.cell_id = ?')) { + return [] + } + if (sql.includes('SELECT cell_id, observed_requests FROM relay_cell_runtime')) { + return [{ cell_id: 'cell-a', observed_requests: 0 }] + } + if (sql.includes('LEFT JOIN relay_cell_admission')) { + return [ + { cell_id: 'cell-a', admission_state: 'general' }, + { cell_id: 'cell-b', admission_state: 'general' } + ] + } + if (sql.includes('FROM relay_cell_connection_limits')) return [] + return [{ changes: 1 }] + } + + async queryLocked(sql: string, params: unknown[] = []): Promise { + if (sql.includes('FROM relay_assignments')) { + this.locks.push('assignment') + return [ + { + user_id: identity.userId, + relay_host_id: identity.relayHostId, + cell_id: 'cell-b', + assignment_epoch: 1, + lease_expires_at: 100, + last_activity_at: 100, + reserved_controls: 0, + reserved_splices: 0, + reserved_invites: 0, + pending_installs: 0, + pending_confirmations: 0, + migration_leases: 0 + } + ] + } + if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') { + this.locks.push('cell-inventory') + return [cellRow('cell-a', 0), cellRow('cell-b', 1)] + } + if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) { + const cellId = String(params[0]) + this.locks.push(cellId) + return [cellRow(cellId, cellId === 'cell-b' ? 1 : 0)] + } + return [] + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await operation(this) + } + + async close(): Promise {} +} + +class AggregateCleanupDatabase implements RelayDatabase { + failIfUnavailable: boolean | null = null + + async query(): Promise { + return [{ changes: 1 }] + } + + async queryLocked( + sql: string, + _params: unknown[] = [], + options: RelayLockOptions = {} + ): Promise { + if (sql.includes('FROM relay_assignments WHERE lease_expires_at')) { + this.failIfUnavailable = options.failIfUnavailable ?? false + } + return [] + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await operation(this) + } + + async close(): Promise {} +} + +class HeartbeatLockDatabase implements RelayDatabase { + readonly locks: string[] = [] + readonly reservationCleanupTransactions: number[] = [] + legacyHeartbeatWritten = false + private transactionNumber = 0 + private activeTransaction = 0 + + constructor( + private readonly cleanupIncarnation = '11111111-1111-4111-8111-111111111111' + ) {} + + async query(sql: string, params: unknown[] = []): Promise { + if (sql.includes('SELECT region FROM relay_cell_regions')) { + return [{ cell_id: String(params[0]), region: 'us-central1' }] + } + if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) { + return [cellRow('cell-a', 0)] + } + if (sql.includes('UPDATE relay_cells SET observed_requests')) { + this.legacyHeartbeatWritten = true + } + if (sql.includes('UPDATE relay_control_connection_reservations')) { + this.reservationCleanupTransactions.push(this.activeTransaction) + } + return [{ changes: 1 }] + } + + async queryLocked(sql: string): Promise { + if (sql.includes('FROM relay_cells')) { + this.locks.push('cell') + return [cellRow('cell-a', 0)] + } + if (sql.includes('FROM relay_cell_runtime')) { + this.locks.push('runtime') + return this.activeTransaction === 2 + ? [{ cell_incarnation: this.cleanupIncarnation }] + : [] + } + if (sql.includes('FROM relay_cell_connection_limits')) { + this.locks.push('connection-limit') + return [{ hard_cap: 600, unobserved_bound: 99 }] + } + if (sql.includes('FROM relay_cell_connection_snapshots')) { + this.locks.push('snapshot') + return this.activeTransaction === 2 + ? [{ cell_incarnation: this.cleanupIncarnation, inclusion_watermark: 0 }] + : [] + } + return [] + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + this.activeTransaction = ++this.transactionNumber + const result = await operation(this) + this.activeTransaction = 0 + return result + } + + async close(): Promise {} +} + +class NewAssignmentLockDatabase implements RelayDatabase { + readonly inventoryLocks: string[] = [] + private generalLockFailed = false + + constructor( + private failGeneralOnce = false, + private readonly assignmentAppearsAfterFailure = false + ) {} + + async query(sql: string, params: unknown[] = []): Promise { + if (sql.includes('SELECT cell_id, region FROM relay_cell_regions')) { + return [ + { cell_id: 'cell-existing', region: 'us-central1' }, + { cell_id: 'cell-general', region: 'us-central1' } + ] + } + if (sql.includes('SELECT region FROM relay_cell_regions')) { + return [{ cell_id: String(params[0]), region: 'us-central1' }] + } + if (sql.includes('SELECT cell_id, observed_requests FROM relay_cell_runtime')) { + return [{ cell_id: 'cell-general', observed_requests: 0 }] + } + if (sql.includes('LEFT JOIN relay_cell_admission')) { + return [{ cell_id: 'cell-general', admission_state: 'general' }] + } + if (sql.includes('JOIN relay_cell_runtime') && sql.includes('cell.cell_id = ?')) { + return [{ cell_id: 'cell-existing' }] + } + if (sql.includes('FROM relay_cell_connection_limits')) { + return [ + { + cell_id: 'cell-general', + hard_cap: 600, + unobserved_bound: 99, + enforced_connection_units: 0, + outstanding_reservations: 0, + last_heartbeat_at: 100, + connection_incarnation: 'incarnation-a', + current_incarnation: 'incarnation-a' + } + ] + } + return [{ changes: 1 }] + } + + async queryLocked(sql: string): Promise { + if ( + sql.includes('FROM relay_assignments') && + this.assignmentAppearsAfterFailure && + this.generalLockFailed + ) { + return [ + { + user_id: identity.userId, + relay_host_id: identity.relayHostId, + cell_id: 'cell-existing', + assignment_epoch: 1, + lease_expires_at: 100, + last_activity_at: 100, + reserved_controls: 1, + reserved_splices: 0, + reserved_invites: 0, + pending_installs: 0, + pending_confirmations: 0, + migration_leases: 0 + } + ] + } + if (sql.includes('SELECT cell_id FROM relay_cell_admission')) { + this.inventoryLocks.push('general') + if (this.failGeneralOnce) { + this.failGeneralOnce = false + this.generalLockFailed = true + throw new Error('database_lock_unavailable') + } + return [cellRow('cell-general', 0)] + } + if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') { + this.inventoryLocks.push('all') + return [cellRow('cell-existing', 0), cellRow('cell-general', 0)] + } + if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) { + return [cellRow('cell-general', 0)] + } + return [] + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await operation(this) + } + + async close(): Promise {} +} + +function cellRow(cellId: string, reservedRequests: number): SqlRow { + return { + cell_id: cellId, + cell_url: `https://${cellId}.example.com`, + enabled: 1, + capacity_requests: 10, + reserved_requests: reservedRequests, + observed_requests: 0 + } +} + +describe('RelayAssignmentStore activity lock order', () => { + it('updates the cell only after assignment activity during release', async () => { + const database = new LockOrderDatabase(false, 100) + const store = new RelayAssignmentStore(database, () => 100) + + await expect(store.releaseActivity(identity, activityId)).resolves.toBe(true) + expect(database.lockedTables).toEqual(['assignment', 'activity', 'cell']) + }) + + it('updates the cell only after inserting new activity', async () => { + const database = new LockOrderDatabase(false, 100, false) + const store = new RelayAssignmentStore(database, () => 100) + + await expect( + store.acquireActivity(identity, { activityId, kind: 'splice', cellId: 'cell-a' }) + ).resolves.toBeUndefined() + expect(database.lockedTables).toEqual(['assignment', 'activity', 'cell']) + }) + + it('rechecks an expired candidate after taking the assignment lock', async () => { + const database = new LockOrderDatabase(true, 101) + const store = new RelayAssignmentStore(database, () => 100) + + await expect(store.releaseExpiredActivityLeases()).resolves.toBe(0) + expect(database.lockedTables).toEqual(['assignment', 'activity']) + }) + + it('fails fast before aggregate cleanup waits on mixed-version assignment rows', async () => { + const database = new AggregateCleanupDatabase() + const store = new RelayAssignmentStore(database, () => 100) + + await expect(store.releaseExpiredActivity()).resolves.toBe(0) + expect(database.failIfUnavailable).toBe(true) + }) + + it('releases the placement capacity lock before heartbeat reservation cleanup', async () => { + const database = new HeartbeatLockDatabase() + const store = new RelayAssignmentStore(database, () => 100, { requireLiveCells: true }) + + await store.recordCellHeartbeat({ + cellId: 'cell-a', + cellUrl: 'https://cell-a.example.com', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionHardCap: 600, + connectionUnobservedBound: 99 + }) + + expect(database.locks).toEqual([ + 'cell', + 'runtime', + 'connection-limit', + 'snapshot', + 'runtime', + 'snapshot' + ]) + expect(database.legacyHeartbeatWritten).toBe(false) + expect(database.reservationCleanupTransactions).toEqual([2, 2, 2]) + }) + + it('does not let an old heartbeat clean replacement-incarnation reservations', async () => { + const database = new HeartbeatLockDatabase('22222222-2222-4222-8222-222222222222') + const store = new RelayAssignmentStore(database, () => 100, { requireLiveCells: true }) + + await store.recordCellHeartbeat({ + cellId: 'cell-a', + cellUrl: 'https://cell-a.example.com', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionHardCap: 600, + connectionUnobservedBound: 99 + }) + + expect(database.reservationCleanupTransactions).toEqual([]) + }) + + it('locks only general-admission inventory for a brand-new assignment', async () => { + const database = new NewAssignmentLockDatabase() + const store = new RelayAssignmentStore(database, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45 + }) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-general', + assignmentEpoch: 1 + }) + expect(database.inventoryLocks).toEqual(['general']) + }) + + it('keeps a brand-new assignment retry scoped to general admission', async () => { + const database = new NewAssignmentLockDatabase(true) + const store = new RelayAssignmentStore(database, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45 + }) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-general', + assignmentEpoch: 1 + }) + expect(database.inventoryLocks).toEqual(['general', 'general']) + }) + + it('restarts with full inventory if an assignment appears during a general retry', async () => { + const database = new NewAssignmentLockDatabase(true, true) + const store = new RelayAssignmentStore(database, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45 + }) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-existing', + assignmentEpoch: 1 + }) + expect(database.inventoryLocks).toEqual(['general', 'general', 'all']) + }) + + it('probes the sticky cell before locking full inventory for dead-cell reassignment', async () => { + const database = new ReassignmentLockOrderDatabase() + const store = new RelayAssignmentStore(database, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45 + }) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-a', + assignmentEpoch: 2 + }) + expect(database.locks).toEqual([ + 'assignment', + 'cell-b', + 'assignment', + 'cell-inventory', + 'cell-b', + 'cell-a' + ]) + }) +}) diff --git a/cloud/apps/relay/src/assignment-store.test.ts b/cloud/apps/relay/src/assignment-store.test.ts new file mode 100644 index 00000000000..e9d27a83db4 --- /dev/null +++ b/cloud/apps/relay/src/assignment-store.test.ts @@ -0,0 +1,4471 @@ +import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract' +import { createHash } from 'node:crypto' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RelayCellConfig } from './config.js' +import { RelayAssignmentStore, STRANDED_MIGRATION_ABANDON_MS } from './assignment-store.js' +import { + encodeMembership, + type CellAdmissionMembership +} from './cell-admission-selector.js' +import { + openInMemoryRelayDatabase, + type RelayDatabase, + type RelayLockOptions, + type RelayTransactionOptions, + type SqlRow +} from './database.js' + +const CELLS: RelayCellConfig[] = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 2 } +] +const NO_EXPIRED_EVACUATION_DIAGNOSTICS = { + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 +} +const NO_REGISTERED_EVACUATION_DIAGNOSTICS = { + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 0 +} +const NO_ACTIVE_MIGRATION_LEASE = { + oldestExpiresAt: null, + oldestRemainingMs: null +} +const FENCE_PLAN_BINDING = { + planObjectName: + 'terraform/state/relay-fence-plans/production/22222222-2222-4222-8222-222222222222.tfplan', + planObjectGeneration: '123456789', + varFileSha256: 'c'.repeat(64), + terraformStateLineage: '33333333-3333-4333-8333-333333333333', + terraformStateSerial: 7, + terraformStateObjectGeneration: '987654321', + terraformStateObjectSha256: 'd'.repeat(64), + requestReason: + 'orca-relay-fence/22222222-2222-4222-8222-222222222222' +} +const FENCE_INVOCATION_ID = '55555555-5555-4555-8555-555555555555' +const FENCE_INVOCATION_REASON = + `${FENCE_PLAN_BINDING.requestReason}/${FENCE_INVOCATION_ID}` + +async function applyGenerationZeroSelector( + store: RelayAssignmentStore, + input: { + attemptId: string + membership: CellAdmissionMembership + } +) { + const current = (await store.inspectCellAdmissionSelector()).selector.membership + return await store.applyCellAdmissionSelector({ + ...input, + expectedGeneration: 0, + expectedMembershipSha256: createHash('sha256') + .update(encodeMembership(current)) + .digest('hex') + }) +} + +function cellFenceEvidence(attemptId = '22222222-2222-4222-8222-222222222222') { + return { + attemptId, + environment: 'production' as const, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + ...FENCE_PLAN_BINDING + } +} + +class OneShotInventoryFailureDatabase implements RelayDatabase { + readonly retryLocks: string[] = [] + private armed = false + private failed = false + + constructor(private readonly delegate: RelayDatabase) {} + + arm(): void { + this.armed = true + } + + async query(sql: string, params: unknown[] = []): Promise { + return await this.delegate.query(sql, params) + } + + async queryLocked( + sql: string, + params: unknown[] = [], + options: RelayLockOptions = {} + ): Promise { + return await this.lockedQuery(this.delegate, sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await this.delegate.transaction( + async (transaction) => + await operation({ + query: async (sql, params = []) => await transaction.query(sql, params), + queryLocked: async (sql, params = [], options = {}) => + await this.lockedQuery(transaction, sql, params, options), + transaction: async (nested) => await transaction.transaction(nested), + close: async () => {} + }) + ) + } + + async close(): Promise { + await this.delegate.close() + } + + private async lockedQuery( + database: RelayDatabase, + sql: string, + params: unknown[], + options: RelayLockOptions + ): Promise { + const inventory = sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC' + if (this.armed && inventory && options.failIfUnavailable) { + this.armed = false + this.failed = true + this.retryLocks.push('inventory-nowait-failed') + throw new Error('database_lock_unavailable') + } + if (this.failed && inventory) this.retryLocks.push('inventory-first') + if (this.failed && sql.includes('FROM relay_assignments')) { + this.retryLocks.push(options.failIfUnavailable ? 'assignment-nowait' : 'assignment') + } + return await database.queryLocked(sql, params, options) + } +} + +class RepeatedDrainAccountingFailureDatabase implements RelayDatabase { + refreshAttempts = 0 + private armed = false + private failuresRemaining = 0 + + constructor(private readonly delegate: RelayDatabase) {} + + arm(failures: number): void { + this.armed = true + this.failuresRemaining = failures + this.refreshAttempts = 0 + } + + async query(sql: string, params: unknown[] = []): Promise { + return await this.delegate.query(sql, params) + } + + async queryLocked( + sql: string, + params: unknown[] = [], + options: RelayLockOptions = {} + ): Promise { + return await this.lockedQuery(this.delegate, sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await this.delegate.transaction( + async (transaction) => + await operation({ + query: async (sql, params = []) => await transaction.query(sql, params), + queryLocked: async (sql, params = [], options = {}) => + await this.lockedQuery(transaction, sql, params, options), + transaction: async (nested) => await transaction.transaction(nested), + close: async () => {} + }) + ) + } + + async close(): Promise { + await this.delegate.close() + } + + private async lockedQuery( + database: RelayDatabase, + sql: string, + params: unknown[], + options: RelayLockOptions + ): Promise { + const drainRefresh = + sql.includes('SELECT migration.*') && + sql.includes('WHERE migration.source_cell_id = ?') && + !sql.includes('source_cell_incarnation') + if (this.armed && drainRefresh) { + this.refreshAttempts++ + if (this.failuresRemaining > 0) { + this.failuresRemaining-- + throw new Error('migration_activity_accounting_mismatch') + } + } + return await database.queryLocked(sql, params, options) + } +} + +// Runs a side effect between the sticky lane and the placement lane, the window +// in which a host can acquire control after the sticky read called it dormant. +class SeedBetweenAssignmentLanesDatabase implements RelayDatabase { + private transactions = 0 + private armed = false + + constructor( + private readonly delegate: RelayDatabase, + private readonly seed: () => Promise + ) {} + + arm(): void { + this.armed = true + this.transactions = 0 + } + + async query(sql: string, params: unknown[] = []): Promise { + return await this.delegate.query(sql, params) + } + + async queryLocked( + sql: string, + params: unknown[] = [], + options: RelayLockOptions = {} + ): Promise { + return await this.delegate.queryLocked(sql, params, options) + } + + async transaction( + operation: (transaction: RelayDatabase) => Promise, + options?: RelayTransactionOptions + ): Promise { + if (this.armed && ++this.transactions === 2) await this.seed() + return await this.delegate.transaction(operation, options) + } + + async close(): Promise { + await this.delegate.close() + } +} + +describe('RelayAssignmentStore', () => { + let database: RelayDatabase | undefined + + afterEach(async () => await database?.close()) + + async function setup(now: () => number, cells = CELLS): Promise { + database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, now) + await store.reconcileCells(cells) + return store + } + + async function setupWithHeartbeats( + now: () => number, + cells = CELLS + ): Promise { + database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells(cells) + return store + } + + async function heartbeat( + store: RelayAssignmentStore, + cell = CELLS[0]!, + input: { + incarnation?: string + startedAt?: number + ready?: boolean + observedRequests?: number + } = {} + ): Promise { + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: input.incarnation ?? '11111111-1111-4111-8111-111111111111', + startedAt: input.startedAt ?? 50, + ready: input.ready ?? true, + observedRequests: input.observedRequests ?? 0, + ...(cell.connectionHardCap === undefined + ? {} + : { + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionHardCap: cell.connectionHardCap, + connectionUnobservedBound: cell.connectionUnobservedBound + }) + }) + } + + it('admits only cells with a fresh ready heartbeat', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + + await expect(store.assign(identity)).rejects.toThrow('relay_capacity_exhausted') + await heartbeat(store, CELLS[0]!, { ready: false }) + await expect(store.assign(identity)).rejects.toThrow('relay_capacity_exhausted') + await heartbeat(store) + expect((await store.assign(identity)).cellId).toBe('cell-a') + now += 45_001 + expect(await store.resolve(identity)).toBeNull() + }) + + it('fences stale incarnations and origin mismatches', async () => { + const store = await setupWithHeartbeats(() => 100, [CELLS[0]!]) + await heartbeat(store, CELLS[0]!, { startedAt: 50 }) + await expect( + heartbeat(store, { ...CELLS[0]!, url: 'https://other.example.com' }) + ).rejects.toThrow('cell_origin_mismatch') + await expect( + heartbeat(store, CELLS[0]!, { + incarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 50 + }) + ).rejects.toThrow('stale_cell_incarnation') + await heartbeat(store, CELLS[0]!, { + incarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 51 + }) + await expect(heartbeat(store, CELLS[0]!, { startedAt: 50 })).rejects.toThrow( + 'stale_cell_incarnation' + ) + }) + + it('records receipt-relative legacy drain states and pins post-send migrations', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now) + await heartbeat(store, CELLS[0]!) + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await database!.query( + `UPDATE relay_assignments SET migration_leases = migration_leases + 1 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + const incarnation = '11111111-1111-4111-8111-111111111111' + const attemptId = '33333333-3333-4333-8333-333333333333' + const traceValue = '44444444-4444-4444-8444-444444444444' + + await expect( + store.prepareCellDrainAttempt({ + attemptId, + cellId: 'cell-a', + cellIncarnation: incarnation, + traceValue, + plannedGraceMs: 120_000 + }) + ).resolves.toMatchObject({ state: 'prepared', shouldSend: false }) + expect( + await database!.query(`SELECT * FROM relay_post_drain_migration_pins`) + ).toEqual([]) + await expect( + store.prepareCellDrainRecovery({ + attemptId, + cellId: 'cell-a', + cellIncarnation: incarnation + }) + ).resolves.toMatchObject({ + shouldSend: false, + preparedAttempt: { + attemptId, + state: 'prepared', + traceValue, + plannedGraceMs: 120_000 + } + }) + + await expect( + store.beginCellDrainSend({ + attemptId, + cellId: 'cell-a', + cellIncarnation: incarnation + }) + ).resolves.toMatchObject({ + state: 'send-may-have-started', + shouldSend: true, + sendPermitExpiresAt: 30_100 + }) + await expect( + database!.query( + `SELECT migration_leases FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).resolves.toEqual([{ migration_leases: 1 }]) + await expect( + store.beginCellDrainSend({ + attemptId, + cellId: 'cell-a', + cellIncarnation: incarnation + }) + ).resolves.toMatchObject({ + state: 'send-may-have-started', + shouldSend: false + }) + await expect( + store.prepareCellDrainRecovery({ + attemptId, + cellId: 'cell-a', + cellIncarnation: incarnation + }) + ).rejects.toThrow('drain_application_receipt_missing') + expect( + await database!.query( + `SELECT drain_attempt_id, source_cell_id, source_cell_incarnation, + target_cell_id, assignment_epoch + FROM relay_post_drain_migration_pins` + ) + ).toEqual([ + { + drain_attempt_id: attemptId, + source_cell_id: 'cell-a', + source_cell_incarnation: incarnation, + target_cell_id: 'cell-b', + assignment_epoch: migration.assignmentEpoch + } + ]) + + now = migration.expiresAt + 1 + expect(await store.abortExpiredEvacuations()).toBe(0) + expect(await store.releaseExpiredActivityLeases()).toBe(1) + expect( + await database!.query( + `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? ORDER BY activity_kind`, + [identity.userId] + ) + ).toEqual([ + { activity_kind: 'control', cell_id: 'cell-b' }, + { activity_kind: 'migration', cell_id: 'cell-b' } + ]) + + now = 200_000 + await expect( + store.recordCellDrainApplicationReceipt({ + attemptId, + cellId: 'cell-a', + cellIncarnation: incarnation, + traceValue, + backendStatus: 200 + }) + ).resolves.toMatchObject({ + state: 'application-receipt', + applicationReceiptAt: 200_000, + retryAfter: 350_000 + }) + await expect( + store.prepareCellDrainRecovery({ attemptId, cellId: 'cell-a', cellIncarnation: incarnation }) + ).rejects.toThrow('drain_recovery_too_early') + now = 350_000 + await expect( + store.prepareCellDrainRecovery({ attemptId, cellId: 'cell-a', cellIncarnation: incarnation }) + ).resolves.toEqual({ + shouldSend: true, + retryAfter: 350_000 + }) + await expect( + store.prepareCellDrainRecovery({ attemptId, cellId: 'cell-a', cellIncarnation: incarnation }) + ).resolves.toEqual({ + shouldSend: false, + retryAfter: 350_000 + }) + }) + + it('recovers a proven drain once after the source cell is replaced', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now) + const oldIncarnation = '11111111-1111-4111-8111-111111111111' + const newIncarnation = '22222222-2222-4222-8222-222222222222' + const newerIncarnation = '55555555-5555-4555-8555-555555555555' + const attempt = { + attemptId: '33333333-3333-4333-8333-333333333333', + cellId: 'cell-a', + cellIncarnation: oldIncarnation, + traceValue: '44444444-4444-4444-8444-444444444444', + plannedGraceMs: 120_000 + } + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + await store.prepareCellDrainAttempt(attempt) + await store.beginCellDrainSend(attempt) + now = 200 + await store.recordCellDrainApplicationReceipt({ + ...attempt, + backendStatus: 200 + }) + + now = 150_200 + await heartbeat(store, CELLS[0]!, { + incarnation: newIncarnation, + startedAt: 51 + }) + await expect( + store.prepareCellDrainRecovery({ + cellId: 'cell-a', + cellIncarnation: newIncarnation + }) + ).resolves.toEqual({ shouldSend: true, retryAfter: 150_200 }) + await expect( + store.prepareCellDrainRecovery({ + cellId: 'cell-a', + cellIncarnation: newIncarnation + }) + ).resolves.toEqual({ shouldSend: false, retryAfter: 150_200 }) + + now = 150_300 + await heartbeat(store, CELLS[0]!, { + incarnation: newerIncarnation, + startedAt: 52 + }) + const concurrentRecoveries = await Promise.all([ + store.prepareCellDrainRecovery({ + cellId: 'cell-a', + cellIncarnation: newerIncarnation + }), + store.prepareCellDrainRecovery({ + cellId: 'cell-a', + cellIncarnation: newerIncarnation + }) + ]) + expect(concurrentRecoveries.map((recovery) => recovery.shouldSend).sort()).toEqual([ + false, + true + ]) + await expect( + store.prepareCellDrainRecovery({ + cellId: 'cell-a', + cellIncarnation: newerIncarnation + }) + ).resolves.toEqual({ shouldSend: false, retryAfter: 150_200 }) + await expect( + database!.query( + `SELECT cell_incarnation FROM relay_cell_drain_recovery_attempts + WHERE drain_attempt_id = ? ORDER BY cell_incarnation`, + [attempt.attemptId] + ) + ).resolves.toEqual([ + { cell_incarnation: newIncarnation }, + { cell_incarnation: newerIncarnation } + ]) + }) + + it('rejects an invalid receipt and a prepared drain from an old incarnation', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now) + const oldIncarnation = '11111111-1111-4111-8111-111111111111' + const newIncarnation = '22222222-2222-4222-8222-222222222222' + const attempt = { + attemptId: '33333333-3333-4333-8333-333333333333', + cellId: 'cell-a', + cellIncarnation: oldIncarnation, + traceValue: '44444444-4444-4444-8444-444444444444', + plannedGraceMs: 120_000 + } + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + await store.prepareCellDrainAttempt(attempt) + now = 150_200 + await heartbeat(store, CELLS[0]!, { + incarnation: newIncarnation, + startedAt: 51 + }) + await expect( + store.prepareCellDrainRecovery({ + cellId: 'cell-a', + cellIncarnation: newIncarnation + }) + ).rejects.toThrow('drain_application_receipt_missing') + + await database!.query( + `UPDATE relay_cell_drain_attempt_states + SET state = 'application-receipt', application_receipt_at = ?, + receipt_cell_incarnation = ?, retry_after = ? + WHERE attempt_id = ?`, + [200, oldIncarnation, 150_200, attempt.attemptId] + ) + await expect( + store.prepareCellDrainRecovery({ + cellId: 'cell-a', + cellIncarnation: newIncarnation + }) + ).rejects.toThrow('drain_application_receipt_missing') + }) + + it('restores an expired registered migration lease before a prepared drain send', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now) + await heartbeat(store, CELLS[0]!) + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + const attempt = { + attemptId: '55555555-5555-4555-8555-555555555555', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: '66666666-6666-4666-8666-666666666666', + plannedGraceMs: 120_000 + } + await store.prepareCellDrainAttempt(attempt) + + now = migration.expiresAt + 1 + expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0) + await heartbeat(store, CELLS[0]!) + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + await expect( + store.beginCellDrainSend({ + attemptId: attempt.attemptId, + cellId: attempt.cellId, + cellIncarnation: attempt.cellIncarnation + }) + ).resolves.toMatchObject({ + state: 'send-may-have-started', + shouldSend: true + }) + await expect( + database!.query( + `SELECT activity_id, activity_kind, cell_id, request_units, expires_at + FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).resolves.toContainEqual({ + activity_id: `migration:${migration.assignmentEpoch}`, + activity_kind: 'migration', + cell_id: 'cell-b', + request_units: 1, + expires_at: now + ASSIGNMENT_LIMITS.migrationLeaseMs + }) + }) + + it('refreshes registered migration leases before prepared drain recovery', async () => { + let now = 100 + const cappedCells = CELLS.map((cell) => ({ + ...cell, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 + })) + const store = await setupWithHeartbeats(() => now, cappedCells) + await heartbeat(store, cappedCells[0]!) + await heartbeat(store, cappedCells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + const attempt = { + attemptId: '99999999-9999-4999-8999-999999999999', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + plannedGraceMs: 120_000 + } + await store.prepareCellDrainAttempt(attempt) + + now = migration.expiresAt - 500_000 + await heartbeat(store, cappedCells[0]!) + await heartbeat(store, cappedCells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + await expect(store.prepareCellDrainRecovery(attempt)).resolves.toMatchObject({ + shouldSend: false, + preparedAttempt: { attemptId: attempt.attemptId } + }) + const refreshedExpiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs + await expect( + database!.query( + `SELECT expires_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, migration.assignmentEpoch] + ) + ).resolves.toEqual([{ expires_at: refreshedExpiresAt }]) + await expect( + database!.query( + `SELECT state, timeout_at FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, migration.assignmentEpoch] + ) + ).resolves.toEqual([{ state: 'reserved', timeout_at: refreshedExpiresAt }]) + }) + + it('bounds repeated accounting repair before prepared drain recovery', async () => { + const delegate = await openInMemoryRelayDatabase() + const retryDatabase = new RepeatedDrainAccountingFailureDatabase(delegate) + database = retryDatabase + const store = new RelayAssignmentStore(retryDatabase, () => 100, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells(CELLS) + await heartbeat(store, CELLS[0]!) + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + const attempt = { + attemptId: '99999999-9999-4999-8999-999999999999', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + plannedGraceMs: 120_000 + } + await store.prepareCellDrainAttempt(attempt) + retryDatabase.arm(2) + + await expect(store.prepareCellDrainRecovery(attempt)).resolves.toMatchObject({ + shouldSend: false, + preparedAttempt: { attemptId: attempt.attemptId } + }) + expect(retryDatabase.refreshAttempts).toBe(3) + + retryDatabase.arm(4) + await expect(store.prepareCellDrainRecovery(attempt)).rejects.toThrow( + 'migration_activity_accounting_mismatch' + ) + expect(retryDatabase.refreshAttempts).toBe(4) + }) + + it('retires a pinned migration superseded by a newer authoritative assignment', async () => { + let now = 100 + const cells = [ + ...CELLS, + { + id: 'cell-c', + url: 'https://relay-c.example.com', + capacityRequests: 2 + } + ] + const store = await setupWithHeartbeats(() => now, cells) + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + await heartbeat(store, cells[2]!, { + incarnation: '33333333-3333-4333-8333-333333333333' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + const attempt = { + attemptId: '99999999-9999-4999-8999-999999999999', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + plannedGraceMs: 120_000 + } + await store.prepareCellDrainAttempt(attempt) + await store.beginCellDrainSend(attempt) + const receipt = await store.recordCellDrainApplicationReceipt({ + ...attempt, + backendStatus: 200 + }) + + await database!.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database!.query( + `UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0, + reserved_invites = 0, pending_installs = 0, pending_confirmations = 0, + migration_leases = 0, lease_expires_at = 0, last_activity_at = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id IN (?, ?)`, + ['cell-a', 'cell-b'] + ) + now = ASSIGNMENT_LIMITS.dormantTtlMs + 1 + await heartbeat(store, cells[2]!, { + incarnation: '33333333-3333-4333-8333-333333333333' + }) + const current = await store.rebalanceDormant(identity, 'cell-c') + await store.activateControl(identity, { + cellId: 'cell-c', + assignmentEpoch: current.assignmentEpoch, + generation: 1 + }) + + expect(receipt.retryAfter).toBeLessThanOrEqual(now) + await heartbeat(store, cells[0]!) + await expect(store.prepareCellDrainRecovery(attempt)).resolves.toMatchObject({ + shouldSend: true + }) + await expect( + database!.query( + `SELECT aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, migration.assignmentEpoch] + ) + ).resolves.toEqual([{ aborted_at: now }]) + await expect( + database!.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).resolves.toEqual([ + { cell_id: 'cell-c', assignment_epoch: current.assignmentEpoch } + ]) + await expect( + database!.query( + `SELECT activity_id, cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).resolves.toEqual([{ activity_id: 'control:cell-c:1', cell_id: 'cell-c' }]) + }) + + it('refuses to restore a missing migration lease without durable target registration', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now) + await heartbeat(store, CELLS[0]!) + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + const attempt = { + attemptId: '77777777-7777-4777-8777-777777777777', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: '88888888-8888-4888-8888-888888888888', + plannedGraceMs: 120_000 + } + await store.prepareCellDrainAttempt(attempt) + + now = migration.expiresAt + 1 + expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0) + await heartbeat(store, CELLS[0]!) + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + await expect( + store.beginCellDrainSend({ + attemptId: attempt.attemptId, + cellId: attempt.cellId, + cellIncarnation: attempt.cellIncarnation + }) + ).rejects.toThrow('migration_activity_lease_shape_mismatch') + }) + + it('moves a dead assignment only after a completed exact-incarnation fence attempt', async () => { + let now = 100 + const cappedCells = CELLS.map((cell) => ({ + ...cell, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 + })) + const store = await setupWithHeartbeats(() => now, cappedCells) + await heartbeat(store, cappedCells[0]!) + await heartbeat(store, cappedCells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.setCellEnabled('cell-a', false) + now += 45_001 + await heartbeat(store, cappedCells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + + await store.attestCellFence('cell-a', '11111111-1111-4111-8111-111111111111') + expect( + await database!.query( + `SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + ['cell-a'] + ) + ).toEqual([]) + expect(await store.evacuateDeadCells()).toBe(0) + expect( + await database!.query( + `SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ cell_id: 'cell-a' }]) + + now += 300_001 + await heartbeat(store, cappedCells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const evidence = { + attemptId: '22222222-2222-4222-8222-222222222222', + environment: 'production' as const, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + ...FENCE_PLAN_BINDING + } + await store.prepareCellFenceAttempt(evidence) + await store.bindCellFencePlanGeneration(evidence, evidence.planObjectGeneration) + await store.startCellFenceApply( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON + ) + await store.recordCellFenceOperation( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON, + 'operation-1' + ) + await store.attestCellFenceAttempt(evidence, 'operation-1') + + expect(await store.evacuateDeadCells()).toBe(1) + expect((await store.resolve(identity))?.cellId).toBe('cell-b') + }) + + it('preserves proven non-delivery and permits one fresh full-grace attempt', async () => { + const store = await setupWithHeartbeats(() => 100, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + const first = { + attemptId: '33333333-3333-4333-8333-333333333333', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: '44444444-4444-4444-8444-444444444444', + plannedGraceMs: 120_000 + } + await store.prepareCellDrainAttempt(first) + await store.beginCellDrainSend(first) + await expect(store.proveCellDrainNotDelivered(first)).resolves.toMatchObject({ + state: 'proven-not-delivered', + provenNotDeliveredAt: 100 + }) + await expect( + store.prepareCellDrainAttempt({ + ...first, + attemptId: '55555555-5555-4555-8555-555555555555', + traceValue: '66666666-6666-4666-8666-666666666666' + }) + ).resolves.toMatchObject({ state: 'prepared' }) + }) + + it('binds fence attestation to one durable Terraform attempt', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + const evidence = { + attemptId: '22222222-2222-4222-8222-222222222222', + environment: 'production' as const, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + ...FENCE_PLAN_BINDING + } + const prepared = await store.prepareCellFenceAttempt(evidence) + expect(prepared).toMatchObject({ createdAt: 100, expiresAt: 3_600_100 }) + expect(prepared.planObjectGeneration).toBeUndefined() + await expect( + store.bindCellFencePlanGeneration( + evidence, + evidence.planObjectGeneration + ) + ).resolves.toMatchObject({ + planObjectGeneration: evidence.planObjectGeneration + }) + await expect( + store.bindCellFencePlanGeneration( + evidence, + evidence.planObjectGeneration + ) + ).resolves.toMatchObject({ + planObjectGeneration: evidence.planObjectGeneration + }) + await expect( + store.bindCellFencePlanGeneration( + evidence, + '987654321' + ) + ).rejects.toThrow('cell_fence_plan_generation_mismatch') + for (const changed of [ + { attemptId: '33333333-3333-4333-8333-333333333333' }, + { environment: 'staging' as const }, + { cellId: 'cell-b' }, + { cellIncarnation: '33333333-3333-4333-8333-333333333333' }, + { migName: 'orca-relay-other' }, + { instanceGroup: `${evidence.instanceGroup}-other` }, + { generationIdentity: `${evidence.generationIdentity}-other` }, + { fenceCommit: 'c'.repeat(40) }, + { planSha256: 'c'.repeat(64) } + ]) { + await expect( + store.startCellFenceApply( + { ...evidence, ...changed }, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON + ) + ).rejects.toThrow() + } + await store.startCellFenceApply( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON + ) + await store.recordCellFenceOperation( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON, + 'operation-1' + ) + now += 45_001 + await expect( + store.attestCellFenceAttempt(evidence, 'operation-other') + ).rejects.toThrow('cell_fence_operation_not_attested') + const attested = await store.attestCellFenceAttempt(evidence, 'operation-1') + expect(attested).toMatchObject({ + expiresAt: now + 300_000, + attempt: { ...evidence, gceOperation: 'operation-1', completedAt: now } + }) + await expect(store.attestCellFenceAttempt(evidence, 'operation-1')).resolves.toEqual( + attested + ) + now += 300_001 + await expect( + store.attestCellFenceAttempt(evidence, 'operation-1') + ).resolves.toMatchObject({ + expiresAt: now + 300_000, + attempt: { completedAt: attested.attempt.completedAt } + }) + }) + + it('aborts only a Terraform fence whose apply never started', async () => { + const store = await setupWithHeartbeats(() => 100, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + const evidence = { + attemptId: '22222222-2222-4222-8222-222222222222', + environment: 'production' as const, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + ...FENCE_PLAN_BINDING + } + await store.prepareCellFenceAttempt(evidence) + await store.bindCellFencePlanGeneration( + evidence, + evidence.planObjectGeneration + ) + await expect(store.abortCellFenceAttempt(evidence)).resolves.toMatchObject({ + abortedAt: 100 + }) + await expect( + store.startCellFenceApply( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON + ) + ).rejects.toThrow( + 'cell_fence_attempt_aborted' + ) + }) + + it('adopts a stale disabled legacy fence only when no durable attempt exists', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + now += 45_001 + + await expect( + store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111') + ).resolves.toBe(now + 300_000) + await expect( + database!.query( + `SELECT cell_id, cell_incarnation FROM relay_cell_fences WHERE cell_id = ?`, + ['cell-a'] + ) + ).resolves.toEqual([ + { + cell_id: 'cell-a', + cell_incarnation: '11111111-1111-4111-8111-111111111111' + } + ]) + await expect( + database!.query( + `SELECT cell_id, cell_incarnation + FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + ['cell-a'] + ) + ).resolves.toEqual([]) + await store.commitLegacyCellFenceAdoption( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + await expect( + database!.query( + `SELECT cell_id, cell_incarnation + FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + ['cell-a'] + ) + ).resolves.toEqual([ + { + cell_id: 'cell-a', + cell_incarnation: '11111111-1111-4111-8111-111111111111' + } + ]) + + now += 1 + await expect( + store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111') + ).resolves.toBe(now + 300_000) + await store.commitLegacyCellFenceAdoption( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + await expect( + database!.query( + `SELECT attested_at, expires_at + FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + ['cell-a'] + ) + ).resolves.toEqual([{ attested_at: now, expires_at: now + 300_000 }]) + + await heartbeat(store) + await expect( + database!.query( + `SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + ['cell-a'] + ) + ).resolves.toEqual([]) + }) + + it.each(['active', 'completed', 'aborted'] as const)( + 'rejects legacy adoption after a %s durable attempt', + async (status) => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + const evidence = cellFenceEvidence() + await store.prepareCellFenceAttempt(evidence) + if (status === 'aborted') await store.abortCellFenceAttempt(evidence) + if (status === 'completed') { + await store.bindCellFencePlanGeneration(evidence, evidence.planObjectGeneration) + await store.startCellFenceApply( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON + ) + await store.recordCellFenceOperation( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON, + 'operation-1' + ) + } + now += 45_001 + if (status === 'completed') { + await store.attestCellFenceAttempt(evidence, 'operation-1') + } + + await expect( + store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111') + ).rejects.toThrow('legacy_cell_fence_attempt_exists') + } + ) + + it('serializes legacy adoption against durable attempt preparation', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + now += 45_001 + + const results = await Promise.allSettled([ + store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111'), + store.prepareCellFenceAttempt(cellFenceEvidence()) + ]) + expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1) + const attempts = await database!.query( + `SELECT COUNT(*) AS count FROM relay_cell_fence_attempts WHERE cell_id = ?`, + ['cell-a'] + ) + const fences = await database!.query( + `SELECT COUNT(*) AS count FROM relay_cell_fences WHERE cell_id = ?`, + ['cell-a'] + ) + const adoptions = await database!.query( + `SELECT COUNT(*) AS count FROM relay_cell_legacy_fence_adoptions + WHERE cell_id = ?`, + ['cell-a'] + ) + expect(Number(attempts[0]!.count) + Number(fences[0]!.count)).toBe(1) + expect(Number(adoptions[0]!.count)).toBe(0) + }) + + it('rejects an expired durable Terraform fence attempt', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + const evidence = { + attemptId: '22222222-2222-4222-8222-222222222222', + environment: 'production' as const, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + ...FENCE_PLAN_BINDING + } + await store.prepareCellFenceAttempt(evidence) + await store.bindCellFencePlanGeneration( + evidence, + evidence.planObjectGeneration + ) + now += 3_600_001 + await expect( + store.startCellFenceApply( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON + ) + ).rejects.toThrow( + 'cell_fence_attempt_expired' + ) + }) + + it('keeps a started Terraform fence attempt recoverable after its preparation TTL', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + await heartbeat(store) + await store.setCellEnabled('cell-a', false) + const evidence = { + attemptId: '22222222-2222-4222-8222-222222222222', + environment: 'production' as const, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + ...FENCE_PLAN_BINDING + } + await store.prepareCellFenceAttempt(evidence) + await store.bindCellFencePlanGeneration( + evidence, + evidence.planObjectGeneration + ) + await store.startCellFenceApply( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON + ) + now += 3_600_001 + await expect( + store.recordCellFenceOperation( + evidence, + FENCE_INVOCATION_ID, + FENCE_INVOCATION_REASON, + 'operation-1' + ) + ).resolves.toMatchObject({ + attempt: { gceOperation: 'operation-1' }, + invocation: { gceOperation: 'operation-1' } + }) + await expect( + store.prepareCellFenceAttempt({ + ...evidence, + attemptId: '44444444-4444-4444-8444-444444444444', + planObjectName: + 'terraform/state/relay-fence-plans/production/44444444-4444-4444-8444-444444444444.tfplan', + requestReason: + 'orca-relay-fence/44444444-4444-4444-8444-444444444444' + }) + ).rejects.toThrow('cell_fence_attempt_evidence_mismatch') + }) + + it('reports aggregate deployment state and heartbeat freshness without identities', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [CELLS[0]!]) + await heartbeat(store) + const identity = { userId: 'private-user', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({ + activityLeases: 1, + activityRequestUnits: 1, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: 0 + }) + await store.activateControl(identity, { + cellId: 'cell-a', + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + + expect(await store.cellDeploymentStatus('cell-a')).toEqual({ + cellId: 'cell-a', + cellUrl: 'https://relay-a.example.com', + region: 'us-central1', + enabled: true, + admissionState: 'general', + capacityRequests: 2, + reservedRequests: 1, + assignments: 1, + activityLeases: 1, + activityRequestUnits: 1, + restartBlockingActivityLeases: 1, + restartBlockingActivityRequestUnits: 1, + restartBlockingReservedRequests: 1, + outgoingMigrations: 0, + incomingMigrations: 0, + connectionCapacity: null, + runtime: { + cellUrl: 'https://relay-a.example.com', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + lastHeartbeatAt: 100, + heartbeatFresh: true, + regionalRehomeProtocol: 0 + } + }) + now += 45_001 + expect((await store.cellDeploymentStatus('cell-a')).runtime?.heartbeatFresh).toBe(false) + await expect(store.cellDeploymentStatus('missing')).rejects.toThrow('cell_not_found') + }) + + it('keeps concurrent sticky assignment grants restart-safe', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'private-user', relayHostId: 'host000000000001' } + + await Promise.all(Array.from({ length: 20 }, async () => await store.assign(identity))) + + expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({ + activityLeases: 1, + activityRequestUnits: 1, + reservedRequests: 1, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: 0 + }) + }) + + it('classifies activated and non-control activity as restart-blocking', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'private-user', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + for (const kind of ['splice', 'invite', 'install', 'confirmation', 'migration'] as const) { + await store.acquireActivity(identity, { + activityId: `${kind}:test`, + kind, + cellId: assignment.cellId + }) + } + + expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({ + activityLeases: 6, + activityRequestUnits: 7, + reservedRequests: 7, + restartBlockingActivityLeases: 6, + restartBlockingActivityRequestUnits: 7, + restartBlockingReservedRequests: 7 + }) + }) + + it('fails closed for malformed pending controls and unexplained reservations', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 } + ]) + await store.assign({ userId: 'private-user', relayHostId: 'host000000000001' }) + await database!.query( + `UPDATE relay_assignment_activity_leases SET request_units = 2 + WHERE cell_id = ?`, + ['cell-a'] + ) + expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({ + restartBlockingActivityLeases: 1, + restartBlockingActivityRequestUnits: 2, + restartBlockingReservedRequests: 1 + }) + + await database!.query( + `UPDATE relay_assignment_activity_leases SET request_units = 1 WHERE cell_id = ?`, + ['cell-a'] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id = ?`, + ['cell-a'] + ) + expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({ + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: -1 + }) + + await database!.query( + `UPDATE relay_cells SET reserved_requests = 2 WHERE cell_id = ?`, + ['cell-a'] + ) + expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({ + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: 1 + }) + }) + + it('keeps a migration blocking when its target also has a pending control', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'private-user', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.startEvacuation(identity, 'cell-b') + + expect(await store.cellDeploymentStatus('cell-b')).toMatchObject({ + activityLeases: 2, + restartBlockingActivityLeases: 1, + restartBlockingActivityRequestUnits: 1, + restartBlockingReservedRequests: 1, + incomingMigrations: 1 + }) + }) + + it('starts declared candidates disabled without overwriting later operator state', async () => { + const candidate: RelayCellConfig = { + id: 'candidate', + url: 'https://candidate.example.com', + capacityRequests: 20, + initiallyEnabled: false + } + const store = await setup(() => 100, [candidate]) + expect((await store.cellDeploymentStatus('candidate')).enabled).toBe(false) + await store.setCellEnabled('candidate', true) + await store.reconcileCells([candidate]) + expect((await store.cellDeploymentStatus('candidate')).enabled).toBe(true) + }) + + it('moves a stranded host off an existing-only cell that stopped serving it', async () => { + // Why: C3's decommission created an existing-only cell that rejects + // attaches; the #194 pin then loops returning hosts forever while each + // grant refreshes their own activity (issue #225). C3 has no + // connection-limits row and expired leases are cleaned within a + // maintenance cycle, so the only durable evidence is the assignment row: + // a recent grant with no live real activity behind it. + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.setCellEnabled(first.cellId, false) + + // The pin holds while the last grant is young enough to still attach. + now += 10_000 + const pinned = await store.assign(identity) + expect(pinned.cellId).toBe(first.cellId) + + // The grant had ample time to attach and produced nothing live. + now += 61_000 + const moved = await store.assign(identity) + expect(moved.cellId).not.toBe(first.cellId) + expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1) + + // The move is durable: no bounce back to the closed cell. + now += 1_000 + const settled = await store.assign(identity) + expect(settled.cellId).toBe(moved.cellId) + expect(settled.assignmentEpoch).toBe(moved.assignmentEpoch) + }) + + it('keeps a serving existing-only cell pinned for hosts with live activity', async () => { + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.setCellEnabled(first.cellId, false) + // A live claimed control proves the cell still serves this host. + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, 'control', ?, 1, ?, ?)`, + [identity.userId, identity.relayHostId, 'control:live-1', first.cellId, now + 600_000, now] + ) + now += 61_000 + const pinned = await store.assign(identity) + expect(pinned.cellId).toBe(first.cellId) + }) + + it('keeps migration-only cells pinned inside the stranded window', async () => { + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.setCellAdmissionState(first.cellId, 'migration-only') + now += 61_000 + const pinned = await store.assign(identity) + expect(pinned.cellId).toBe(first.cellId) + }) + + it('leaves quiet existing-only assignments to the normal dormancy rule', async () => { + // Why: outside the 15-minute stranded window there is no active retry + // loop to break; ordinary returns stay pinned until 24h dormancy. + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.setCellEnabled(first.cellId, false) + now += 20 * 60_000 + const pinned = await store.assign(identity) + expect(pinned.cellId).toBe(first.cellId) + }) + + it('reassigns an active assignment from an uncapped stale cell without a fence', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now) + await heartbeat(store, CELLS[0]!) + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.changeActivity(identity, 'invite', 1) + await database!.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + 'cell-b', + 'cell-a', + -1, + 0, + 0, + 1, + 90_100, + now, + now + ] + ) + await database!.query( + `INSERT INTO relay_post_drain_migration_pins + (user_id, relay_host_id, assignment_epoch, drain_attempt_id, + source_cell_id, source_cell_incarnation, target_cell_id, + target_cell_incarnation, source_request_units, target_reserved_units, + pinned_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + 0, + '33333333-3333-4333-8333-333333333333', + 'cell-b', + '22222222-2222-4222-8222-222222222222', + 'cell-a', + '11111111-1111-4111-8111-111111111111', + 0, + 1, + now + ] + ) + + now += 45_001 + await heartbeat(store, CELLS[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222' + }) + expect(await store.evacuateDeadCells()).toBe(1) + const replacement = await store.resolve(identity) + expect(replacement).toMatchObject({ + cellId: 'cell-b', + assignmentEpoch: first.assignmentEpoch + 1 + }) + expect( + await database!.query( + `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ activity_kind: 'control', cell_id: 'cell-b' }]) + }) + + it('does not let an unfenced capped cell starve eligible legacy recovery', async () => { + let now = 100 + const cells: RelayCellConfig[] = [ + { + id: 'cell-a', + url: 'https://relay-a.example.com', + capacityRequests: 10, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + for (const cell of cells) await heartbeat(store, cell) + await store.setCellEnabled('cell-b', false) + await store.setCellEnabled('cell-c', false) + const cappedIdentity = { userId: 'a-capped', relayHostId: 'host000000000001' } + expect(await store.assign(cappedIdentity)).toMatchObject({ cellId: 'cell-a' }) + + await store.setCellEnabled('cell-a', false) + await store.setCellEnabled('cell-b', true) + const legacyIdentity = { userId: 'z-legacy', relayHostId: 'host000000000002' } + expect(await store.assign(legacyIdentity)).toMatchObject({ cellId: 'cell-b' }) + await store.setCellEnabled('cell-c', true) + + now += 45_001 + await heartbeat(store, cells[2]!) + expect(await store.evacuateDeadCells(1)).toBe(1) + expect(await store.resolve(legacyIdentity)).toMatchObject({ cellId: 'cell-c' }) + expect( + await database!.query( + `SELECT cell_id FROM relay_assignments WHERE user_id = ?`, + [cappedIdentity.userId] + ) + ).toEqual([{ cell_id: 'cell-a' }]) + }) + + it('refuses evacuation into a cell without a fresh ready heartbeat', async () => { + const store = await setupWithHeartbeats(() => 100) + await heartbeat(store, CELLS[0]!) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + + await expect(store.startEvacuation(identity, 'cell-b')).rejects.toThrow( + 'target_cell_unavailable' + ) + }) + + it('keeps an active assignment sticky without increasing its epoch or reservation', async () => { + const store = await setup(() => 100) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const second = await store.assign(identity) + + expect(second).toEqual(first) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 }) + }) + + it('selects the least-loaded cell with a stable cell-id tie break', async () => { + const store = await setup(() => 100) + const first = await store.assign({ userId: 'user-a', relayHostId: 'host000000000001' }) + const second = await store.assign({ userId: 'user-b', relayHostId: 'host000000000002' }) + + expect([first.cellId, second.cellId]).toEqual(['cell-a', 'cell-b']) + }) + + it('admits exactly through the configured capacity boundary', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 } + ]) + await store.assign({ userId: 'user-a', relayHostId: 'host000000000001' }) + await store.assign({ userId: 'user-b', relayHostId: 'host000000000002' }) + + await expect( + store.assign({ userId: 'user-c', relayHostId: 'host000000000003' }) + ).rejects.toThrow('relay_capacity_exhausted') + expect(await cellReservations(database!)).toEqual({ 'cell-a': 2 }) + }) + + it('does not oversubscribe when assignments race', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 } + ]) + const results = await Promise.allSettled( + Array.from({ length: 8 }, (_, index) => + store.assign({ userId: `user-${index}`, relayHostId: `host${String(index).padStart(12, '0')}` }) + ) + ) + + expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(2) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 2 }) + }) + + it('reassigns only after all activity expires and the dormant TTL elapses', async () => { + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.changeActivity(identity, 'invite', 1) + await store.changeActivity(identity, 'control', -1) + + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + await store.releaseExpiredActivityLeases() + await store.releaseExpiredActivity() + await database!.query(`UPDATE relay_cells SET observed_requests = 2 WHERE cell_id = ?`, [ + first.cellId + ]) + now += ASSIGNMENT_LIMITS.dormantTtlMs + const reassigned = await store.assign(identity) + + expect(reassigned.cellId).not.toBe(first.cellId) + expect(reassigned.assignmentEpoch).toBe(first.assignmentEpoch + 1) + }) + + it('will not normally move an assignment while any durable activity remains', async () => { + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.changeActivity(identity, 'migration', 1) + await store.changeActivity(identity, 'control', -1) + await database!.query(`UPDATE relay_cells SET observed_requests = 2 WHERE cell_id = ?`, [ + first.cellId + ]) + now += ASSIGNMENT_LIMITS.dormantTtlMs + 1 + + expect(await store.assign(identity)).toMatchObject({ + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch + }) + }) + + it('releases every expired activity reservation without going negative', async () => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + for (const kind of ['splice', 'invite', 'install', 'confirmation', 'migration'] as const) { + await store.changeActivity(identity, kind, 1) + } + expect(await cellReservations(database!)).toEqual({ 'cell-a': 7 }) + + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + expect(await store.releaseExpiredActivityLeases()).toBe(1) + expect(await store.releaseExpiredActivity()).toBe(1) + expect(await store.releaseExpiredActivity()).toBe(0) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0 }) + }) + + it('isolates assignments by host and verifies both cell and epoch', async () => { + const store = await setup(() => 100) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + + await expect( + store.verifyCellAssignment({ ...identity, cellId: assignment.cellId, assignmentEpoch: 1 }) + ).resolves.toBe(true) + await expect( + store.verifyCellAssignment({ ...identity, cellId: 'cell-b', assignmentEpoch: 1 }) + ).resolves.toBe(false) + await expect( + store.verifyCellAssignment({ ...identity, cellId: assignment.cellId, assignmentEpoch: 2 }) + ).resolves.toBe(false) + await expect( + store.resolve({ userId: 'other', relayHostId: identity.relayHostId }) + ).resolves.toBeNull() + }) + + it('converts the director reservation into an idempotent active-control lease', async () => { + const store = await setup(() => 100) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + + const activityId = await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + + expect(activityId).toBe(`control:${assignment.cellId}:1`) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 }) + const leases = await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases` + ) + expect(leases).toEqual([{ activity_id: activityId }]) + }) + + it('transactionally supersedes older controls on only the same cell', async () => { + const cells = [ + { + id: 'cell-a', + url: 'https://relay-a.example.com', + capacityRequests: 10 + }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const store = await setup(() => 100, cells) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.setCellEnabled('cell-b', false) + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-b', true) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, 'control', 'cell-b', 1, 90100, 100), + (?, ?, ?, 'control', 'cell-b', 1, 90100, 100)`, + [ + identity.userId, + identity.relayHostId, + 'control:cell-b:2', + identity.userId, + identity.relayHostId, + 'control:cell-b:3' + ] + ) + const latest = await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 4 + }) + + expect( + await database!.query( + `SELECT activity_id, cell_id FROM relay_assignment_activity_leases + WHERE activity_kind = 'control' ORDER BY cell_id` + ) + ).toEqual([ + { activity_id: 'control:cell-a:1', cell_id: 'cell-a' }, + { activity_id: latest, cell_id: 'cell-b' } + ]) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 2 }) + expect( + await database!.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 2 }]) + }) + + it('re-reserves a sticky grant for a host holding no control lease', async () => { + const store = await setup(() => 100) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + const control = await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.releaseActivity(identity, control) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 }) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch + }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 }) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases` + ) + ).toEqual([{ activity_id: 'control-pending:1' }]) + expect( + await database!.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 1 }]) + }) + + it('re-pins a host that took control after the sticky lane read it dormant', async () => { + const now = 100 + const delegate = await openInMemoryRelayDatabase() + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const seeded = new SeedBetweenAssignmentLanesDatabase(delegate, async () => { + await delegate.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, 'control:cell-a:1', 'control', 'cell-a', 1, ?, ?), + (?, ?, 'control-pending:1', 'control', 'cell-a', 1, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now, + identity.userId, + identity.relayHostId, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now + ] + ) + await delegate.query( + `UPDATE relay_assignments SET reserved_controls = 2 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await delegate.query( + `UPDATE relay_cells SET reserved_requests = 2 WHERE cell_id = 'cell-a'` + ) + }) + database = seeded + const store = new RelayAssignmentStore(seeded, () => now) + await store.reconcileCells(CELLS) + await delegate.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, 'cell-a', 1, ?, ?, 0, 0, 0, 0, 0, 0)`, + [identity.userId, identity.relayHostId, now, now - ASSIGNMENT_LIMITS.dormantTtlMs] + ) + seeded.arm() + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-a', + assignmentEpoch: 1 + }) + expect( + await delegate.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 2 }]) + expect(await cellReservations(delegate)).toEqual({ 'cell-a': 2, 'cell-b': 0 }) + expect( + await delegate.query( + `SELECT lease_expires_at, last_activity_at FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { lease_expires_at: now + ASSIGNMENT_LIMITS.activityLeaseMs, last_activity_at: now } + ]) + }) + + it('reserves control on the pinned cell when the only lease sits on another', async () => { + const now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await database!.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, 'cell-b', 2, ?, ?, 1, 0, 0, 0, 0, 0)`, + [ + identity.userId, + identity.relayHostId, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now + ] + ) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, 'control:cell-a:1', 'control', 'cell-a', 1, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now + ] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = 'cell-a'` + ) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-b', + assignmentEpoch: 2 + }) + expect( + await database!.query( + `SELECT activity_id, cell_id FROM relay_assignment_activity_leases + ORDER BY activity_id ASC` + ) + ).toEqual([ + { activity_id: 'control-pending:2', cell_id: 'cell-b' }, + { activity_id: 'control:cell-a:1', cell_id: 'cell-a' } + ]) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 1 }) + expect( + await database!.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 2 }]) + }) + + it('mints a pending control when the only lease is an older epoch pending', async () => { + const now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await database!.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, 'cell-a', 3, ?, ?, 1, 0, 0, 0, 0, 0)`, + [ + identity.userId, + identity.relayHostId, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now + ] + ) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, 'control-pending:1', 'control', 'cell-a', 1, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now + ] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = 'cell-a'` + ) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-a', + assignmentEpoch: 3 + }) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases + ORDER BY activity_id ASC` + ) + ).toEqual([{ activity_id: 'control-pending:1' }, { activity_id: 'control-pending:3' }]) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 2, 'cell-b': 0 }) + expect( + await database!.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 2 }]) + }) + + it('re-reserves a re-pinned grant for a host holding no control lease', async () => { + const now = 100 + const delegate = await openInMemoryRelayDatabase() + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const seeded = new SeedBetweenAssignmentLanesDatabase(delegate, async () => { + await delegate.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, 'splice:connection-1', 'splice', 'cell-a', 1, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now + ] + ) + await delegate.query( + `UPDATE relay_assignments SET reserved_splices = 1 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await delegate.query( + `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = 'cell-a'` + ) + }) + database = seeded + const store = new RelayAssignmentStore(seeded, () => now) + await store.reconcileCells(CELLS) + await delegate.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, 'cell-a', 1, ?, ?, 0, 0, 0, 0, 0, 0)`, + [identity.userId, identity.relayHostId, now, now - ASSIGNMENT_LIMITS.dormantTtlMs] + ) + seeded.arm() + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: 'cell-a', + assignmentEpoch: 1 + }) + expect( + await delegate.query( + `SELECT reserved_controls, reserved_splices FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 1, reserved_splices: 1 }]) + expect( + await delegate.query( + `SELECT activity_id FROM relay_assignment_activity_leases + ORDER BY activity_id ASC` + ) + ).toEqual([{ activity_id: 'control-pending:1' }, { activity_id: 'splice:connection-1' }]) + expect(await cellReservations(delegate)).toEqual({ 'cell-a': 2, 'cell-b': 0 }) + }) + + it('extends the lease of a control-holding host on a sticky grant', async () => { + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + + now = 40_000 + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: assignment.cellId + }) + expect( + await database!.query( + `SELECT lease_expires_at, last_activity_at FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { lease_expires_at: now + ASSIGNMENT_LIMITS.activityLeaseMs, last_activity_at: now } + ]) + }) + + // The old absolute write shortened a 15-minute migration lease to the 90s + // grant lease; only the touch's monotonic CASE keeps the longer one now. + it('never shortens a migration lease to the grant lease', async () => { + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + const before = await database!.query( + `SELECT lease_expires_at FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + expect(before).toEqual([{ lease_expires_at: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs }]) + + now = 1000 + await expect(store.assign(identity)).resolves.toMatchObject({ cellId: 'cell-b' }) + expect( + await database!.query( + `SELECT lease_expires_at, last_activity_at FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { lease_expires_at: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs, last_activity_at: now } + ]) + }) + + it('moves an origin-scoped activity reservation without double counting', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.acquireActivity(identity, { + activityId: 'splice:connection-1', + kind: 'splice', + cellId: 'cell-a' + }) + await store.startEvacuation(identity, 'cell-b') + await store.acquireActivity(identity, { + activityId: 'splice:connection-1', + kind: 'splice', + cellId: 'cell-b' + }) + + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 6 }) + await expect(store.releaseActivity(identity, 'splice:connection-1')).resolves.toBe(true) + await expect(store.releaseActivity(identity, 'splice:connection-1')).resolves.toBe(false) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 4 }) + }) + + it('rejects a durable activity before it can exceed cell capacity', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + + await expect( + store.acquireActivity(identity, { + activityId: 'splice:connection-1', + kind: 'splice', + cellId: 'cell-a' + }) + ).rejects.toThrow('relay_capacity_exhausted') + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1 }) + }) + + it('moves active assignments target-first and completes only after source drain', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + + const migration = await store.startEvacuation(identity, 'cell-b') + expect(await store.startEvacuation(identity, 'cell-b')).toEqual(migration) + expect(migration).toMatchObject({ + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + previousEpoch: 1, + assignmentEpoch: 2 + }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 2 }) + + const targetControl = await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: 2, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { cellId: 'cell-b', assignmentEpoch: 2 }) + await expect(store.completeEvacuation(identity, 2)).rejects.toThrow( + 'migration_source_still_active' + ) + await store.releaseActivity(identity, sourceControl) + await store.completeEvacuation(identity, 2) + + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 }) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 }) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases ORDER BY activity_id` + ) + ).toEqual([{ activity_id: targetControl }]) + await expect( + store.acquireActivity(identity, { + activityId: 'splice:late-source', + kind: 'splice', + cellId: 'cell-a' + }) + ).rejects.toThrow('activity_cell_not_authoritative') + }) + + it('automatically completes only after the source runtime is freshly quiescent', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[1]!) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + await store.setCellEnabled('cell-a', false) + await heartbeat(store, cells[0]!, { observedRequests: 1 }) + + expect(await store.completeReadyEvacuations()).toBe(0) + await heartbeat(store, cells[0]!, { observedRequests: 0 }) + now = 150 + await heartbeat(store, cells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 125 + }) + expect(await store.completeReadyEvacuations()).toBe(0) + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 2 + }) + expect(await store.completeReadyEvacuations()).toBe(1) + }) + + it('completes a fenced migration only after the source heartbeat is stale', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + await heartbeat(store, cells[0]!, { observedRequests: 1 }) + await heartbeat(store, cells[1]!) + await store.setCellEnabled('cell-b', false) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-b', true) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + await store.setCellEnabled('cell-a', false) + + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({ + inProgress: 1, + blocked: 1 + }) + now += 45_001 + await heartbeat(store, cells[1]!) + await store.attestCellFence( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({ + inProgress: 0, + completed: 1 + }) + }) + + it('blocks aggregate fenced completion on assignment accounting mismatch', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[1]!) + await store.setCellEnabled('cell-b', false) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-b', true) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + await store.setCellEnabled('cell-a', false) + await database!.query( + `UPDATE relay_assignments SET reserved_splices = reserved_splices + 1 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + now += 45_001 + await heartbeat(store, cells[1]!) + await store.attestCellFence( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({ + inProgress: 1, + blocked: 1 + }) + }) + + it('blocks aggregate fenced completion on an unexpected third-cell lease', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + for (const cell of cells) await heartbeat(store, cell) + await store.setCellEnabled('cell-b', false) + await store.setCellEnabled('cell-c', false) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-b', true) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + await store.setCellEnabled('cell-a', false) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + 'splice:unexpected-cell', + 'splice', + 'cell-c', + 2, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now + ] + ) + await database!.query( + `UPDATE relay_assignments SET reserved_splices = reserved_splices + 1 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = reserved_requests + 2 WHERE cell_id = ?`, + ['cell-c'] + ) + now += 45_001 + await heartbeat(store, cells[1]!) + await store.attestCellFence( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({ + inProgress: 1, + blocked: 1 + }) + }) + + it('rolls back an unregistered expired evacuation with a strictly newer epoch', async () => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + expect(await store.abortExpiredEvacuations()).toBe(1) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-a', assignmentEpoch: 3 }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 }) + }) + + // Why: completion needs an enabled target and expiry rollback needs an unregistered one, so a + // target disabled after it registered satisfies neither and the migration never leaves the table. + async function wedgeRegisteredMigration( + now: () => number, + cells: RelayCellConfig[], + disableTarget = true, + disableSource = true, + releaseSourceControl = true + ): Promise<{ + store: RelayAssignmentStore + identity: { userId: string; relayHostId: string } + }> { + const store = await setupWithHeartbeats(now, cells) + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[1]!) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + const migration = await store.startEvacuation(identity, 'cell-b') + const targetControl = await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + if (releaseSourceControl) await store.releaseActivity(identity, sourceControl) + if (disableSource) await store.setCellEnabled('cell-a', false) + // The desktop leaves the half-migrated target, then an operator disables that cell. + await store.releaseActivity(identity, targetControl) + if (disableTarget) await store.setCellEnabled('cell-b', false) + return { store, identity } + } + + async function insertReservedControlConnection( + identity: { userId: string; relayHostId: string }, + cellId: string, + assignmentEpoch: number, + now: number + ): Promise { + await database!.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch, + cell_id, state, created_at, timeout_at, updated_at) + VALUES ('abandoned-reservation', 'abandoned-key', ?, ?, ?, ?, 'reserved', ?, ?, ?)`, + [identity.userId, identity.relayHostId, assignmentEpoch, cellId, now, now + 60_000, now] + ) + } + + it('reaps an expired migration whose registered target cell was disabled', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store, identity } = await wedgeRegisteredMigration(() => now, cells) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + // A disabled target can never satisfy completion, so waiting cannot help. + expect(await store.completeReadyEvacuations()).toBe(0) + expect(await store.abortExpiredEvacuations()).toBe(1) + // Rollback lands on the disabled source, so the host resolves to nothing and is placed afresh + // by evacuateDeadCells; the point is that the row is retired rather than reaped every tick. + expect(await store.resolve(identity)).toBeNull() + expect(await store.abortExpiredEvacuations()).toBe(0) + }) + + it('leaves a freshly disabled target for supersede-target to recover', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells) + + // Expired, but a disabled target is what supersede-target itself creates, so the operator + // window must stay open; aborting here would fail their run with migration_already_superseded. + now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + expect(await store.abortExpiredEvacuations()).toBe(0) + + now += STRANDED_MIGRATION_ABANDON_MS + expect(await store.abortExpiredEvacuations()).toBe(1) + }) + + it('rolls an abandoned disabled target back while its source remains active', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store, identity } = await wedgeRegisteredMigration( + () => now, + cells, + true, + false, + false + ) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + expect(await store.abortExpiredEvacuations()).toBe(1) + expect( + await database!.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ cell_id: 'cell-a', assignment_epoch: 3 }]) + expect( + await database!.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ completed_at: null, aborted_at: now }]) + }) + + it('starts the abandon window when an old migration target is disabled', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells, false, false) + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + + await store.setCellEnabled('cell-b', false) + expect(await store.abortExpiredEvacuations()).toBe(0) + + now += STRANDED_MIGRATION_ABANDON_MS + 1 + expect(await store.abortExpiredEvacuations()).toBe(1) + }) + + it('reaps an expired migration from a retired source when its target control is gone', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store, identity } = await wedgeRegisteredMigration(() => now, cells, false) + await applyGenerationZeroSelector(store, { + attemptId: 'retired_source_migration_target', + membership: { + existingOnly: ['cell-a'], + migrationOnly: ['cell-b'], + general: [] + } + }) + await insertReservedControlConnection(identity, 'cell-b', 2, now) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + expect(await store.abortExpiredEvacuations()).toBe(1) + expect( + await database!.query( + `SELECT assignment.cell_id, assignment.assignment_epoch, + migration.completed_at, migration.aborted_at + FROM relay_assignments assignment + JOIN relay_assignment_migrations migration + ON migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + WHERE assignment.user_id = ? AND assignment.relay_host_id = ? + AND migration.assignment_epoch = ?`, + [identity.userId, identity.relayHostId, 2] + ) + ).toEqual([ + { + cell_id: 'cell-b', + assignment_epoch: 2, + completed_at: now, + aborted_at: null + } + ]) + expect( + await database!.query( + `SELECT reserved_controls, migration_leases FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 0, migration_leases: 0 }]) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([]) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 }) + expect( + await database!.query( + `SELECT state FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ state: 'released' }]) + }) + + it('reaps a retired-side migration with only a pending target control', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[1]!) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + await applyGenerationZeroSelector(store, { + attemptId: 'retired_source_pending_target', + membership: { + existingOnly: ['cell-a'], + migrationOnly: ['cell-b'], + general: [] + } + }) + await insertReservedControlConnection(identity, 'cell-b', migration.assignmentEpoch, now) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + await database!.query( + `UPDATE relay_assignment_activity_leases SET expires_at = ? + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [now, identity.userId, identity.relayHostId, `control-pending:${migration.assignmentEpoch}`] + ) + expect(await store.abortExpiredEvacuations()).toBe(1) + expect( + await database!.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ cell_id: 'cell-b', assignment_epoch: migration.assignmentEpoch }]) + expect( + await database!.query( + `SELECT reserved_controls, migration_leases FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ reserved_controls: 0, migration_leases: 0 }]) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([]) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 }) + expect( + await database!.query( + `SELECT state FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ state: 'released' }]) + }) + + it('preserves a fresh target grant created after abandoned migration selection', async () => { + let now = 100 + const cells: RelayCellConfig[] = [ + { + id: 'cell-a', + url: 'https://relay-a.example.com', + capacityRequests: 10, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }, + { + id: 'cell-b', + url: 'https://relay-b.example.com', + capacityRequests: 10, + connectionHardCap: 600, + connectionUnobservedBound: 50 + } + ] + const { store, identity } = await wedgeRegisteredMigration(() => now, cells, false) + await applyGenerationZeroSelector(store, { + attemptId: 'retired_source_fresh_target_grant', + membership: { + existingOnly: ['cell-a'], + migrationOnly: ['cell-b'], + general: [] + } + }) + + now += 45_001 + await heartbeat(store, cells[1]!) + await store.adoptLegacyCellFence( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + await store.commitLegacyCellFenceAdoption( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + await heartbeat(store, cells[1]!) + const query = database!.query.bind(database) + let grant: Awaited> | undefined + vi.spyOn(database!, 'query').mockImplementationOnce(async (sql, params) => { + const rows = await query(sql, params) + grant = await store.assign(identity) + return rows + }) + + expect(await store.abortExpiredEvacuations()).toBe(0) + expect(grant).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 }) + expect( + await database!.query( + `SELECT activity_id, expires_at FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, 'control-pending:2'] + ) + ).toEqual([{ activity_id: 'control-pending:2', expires_at: grant!.leaseExpiresAt }]) + expect( + await database!.query( + `SELECT state FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND cell_id = ? AND state = 'reserved'`, + [identity.userId, identity.relayHostId, 2, 'cell-b'] + ) + ).toEqual([{ state: 'reserved' }]) + + await store.activateControl(identity, { + cellId: grant!.cellId, + assignmentEpoch: grant!.assignmentEpoch, + generation: 2 + }) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ activity_id: 'control:cell-b:2' }]) + expect( + await database!.query( + `SELECT completed_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, 2] + ) + ).toEqual([{ completed_at: null }]) + }) + + it('keeps an abandoned target migration open while its source is still active', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store, identity } = await wedgeRegisteredMigration( + () => now, + cells, + false, + true, + false + ) + await applyGenerationZeroSelector(store, { + attemptId: 'retired_source_active', + membership: { + existingOnly: ['cell-a'], + migrationOnly: ['cell-b'], + general: [] + } + }) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + expect(await store.abortExpiredEvacuations()).toBe(0) + expect( + await database!.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ completed_at: null, aborted_at: null }]) + }) + + it('starts the abandon window when an old migration source is retired', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells, false, false) + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + + await store.setCellEnabled('cell-a', false) + expect(await store.abortExpiredEvacuations()).toBe(0) + + now += STRANDED_MIGRATION_ABANDON_MS + 1 + expect(await store.abortExpiredEvacuations()).toBe(1) + }) + + it('reaps after an old source migration lease is refreshed and expires', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells, false) + await applyGenerationZeroSelector(store, { + attemptId: 'retired_source_refreshed_lease', + membership: { + existingOnly: ['cell-a'], + migrationOnly: ['cell-b'], + general: [] + } + }) + now += STRANDED_MIGRATION_ABANDON_MS + ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + await database!.query( + `UPDATE relay_assignment_migrations SET expires_at = ?`, + [now - 1] + ) + + expect(await store.abortExpiredEvacuations()).toBe(1) + }) + + it('keeps a freshly retired target open despite an old retired source', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells, false) + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + + await store.setCellEnabled('cell-b', false) + expect(await store.abortExpiredEvacuations()).toBe(0) + + now += STRANDED_MIGRATION_ABANDON_MS + 1 + expect(await store.abortExpiredEvacuations()).toBe(1) + }) + + it('fails supersession closed when reaping wins after selection', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells) + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + const query = database!.query.bind(database) + vi.spyOn(database!, 'query').mockImplementationOnce(async (sql, params) => { + const rows = await query(sql, params) + expect(await store.abortExpiredEvacuations()).toBe(1) + return rows + }) + + await expect( + store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100) + ).rejects.toThrow('migration_already_superseded') + }) + + it('fails supersession closed when reaping wins after preparation', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells) + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + await store.attestCellFence('cell-b', '11111111-1111-4111-8111-111111111111') + const supersede = store.supersedeRegisteredEvacuation.bind(store) + vi.spyOn(store, 'supersedeRegisteredEvacuation').mockImplementationOnce(async (...args) => { + expect(await store.abortExpiredEvacuations()).toBe(1) + return await supersede(...args) + }) + + await expect( + store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100) + ).rejects.toThrow('migration_already_superseded') + }) + + it('fails supersession closed when reaping wins before an accounting retry', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const { store } = await wedgeRegisteredMigration(() => now, cells) + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[2]!) + await store.attestCellFence('cell-b', '11111111-1111-4111-8111-111111111111') + await database!.query(`UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = ?`, [ + 'cell-c' + ]) + const supersede = store.supersedeRegisteredEvacuation.bind(store) + let attempts = 0 + vi.spyOn(store, 'supersedeRegisteredEvacuation').mockImplementation(async (...args) => { + attempts++ + if (attempts === 2) expect(await store.abortExpiredEvacuations()).toBe(1) + return await supersede(...args) + }) + + await expect( + store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100) + ).rejects.toThrow('migration_already_superseded') + expect(attempts).toBe(2) + }) + + it('reaps a pinned expired migration once its target cell is disabled', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ] + const { store, identity } = await wedgeRegisteredMigration(() => now, cells) + // Drains pin their migrations and nothing ever deletes a pin, so the pin outlives the drain. + await database!.query( + `INSERT INTO relay_post_drain_migration_pins + (user_id, relay_host_id, assignment_epoch, drain_attempt_id, source_cell_id, + source_cell_incarnation, target_cell_id, target_cell_incarnation, + source_request_units, target_reserved_units, pinned_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + 2, + 'attempt-0000', + 'cell-a', + '11111111-1111-4111-8111-111111111111', + 'cell-b', + '11111111-1111-4111-8111-111111111111', + 1, + 1, + now + ] + ) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + expect(await store.completeReadyEvacuations()).toBe(0) + expect(await store.abortExpiredEvacuations()).toBe(1) + expect(await store.abortExpiredEvacuations()).toBe(0) + }) + + it('repairs an expired migration marker from its exact active target control', async () => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: 2, + generation: 1 + }) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: 2, + generation: 1 + }) + expect(await store.abortExpiredEvacuations()).toBe(0) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({ + inProgress: 1, + oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs, + oldestRemainingMs: -1, + targetRegistered: 1, + registeredSourceActive: 1, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 0, + blocked: 0, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + await store.releaseActivity(identity, sourceControl) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({ + inProgress: 0, + ...NO_ACTIVE_MIGRATION_LEASE, + targetRegistered: 0, + ...NO_REGISTERED_EVACUATION_DIAGNOSTICS, + completed: 1, + blocked: 0, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 }) + }) + + it('keeps a registered migration pending while its proven target control is offline', async () => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + const migration = await store.startEvacuation(identity, 'cell-b') + const targetControl = await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 2 + }) + expect( + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + ).toBe(true) + expect(await store.completeReadyEvacuations()).toBe(0) + await store.releaseActivity(identity, sourceControl) + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + expect(await store.completeReadyEvacuations()).toBe(0) + await expect( + store.completeEvacuation(identity, migration.assignmentEpoch) + ).rejects.toThrow('migration_target_not_active') + await store.releaseActivity(identity, targetControl) + expect(await store.completeReadyEvacuations()).toBe(0) + + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({ + inProgress: 1, + oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs, + oldestRemainingMs: + ASSIGNMENT_LIMITS.migrationLeaseMs - ASSIGNMENT_LIMITS.activityLeaseMs - 1, + targetRegistered: 1, + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 1, + completed: 0, + blocked: 1, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 3 + }) + expect(await store.completeReadyEvacuations()).toBe(1) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({ + inProgress: 0, + ...NO_ACTIVE_MIGRATION_LEASE, + targetRegistered: 0, + ...NO_REGISTERED_EVACUATION_DIAGNOSTICS, + completed: 0, + blocked: 0, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 }) + }) + + it('completes a registered migration after its disabled source is proven dead', async () => { + let now = 100 + const store = await setupWithHeartbeats(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + await heartbeat(store, { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }) + await heartbeat(store, { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + now += 45_001 + await heartbeat( + store, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { startedAt: 50 } + ) + await store.attestCellFence( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + + const input = { + assignmentEpoch: migration.assignmentEpoch, + sourceCellId: 'cell-a', + targetCellId: 'cell-b' + } + await expect(store.completeEvacuationFromDeadSource(identity, input)).resolves.toEqual({ + changed: true, + assignmentEpoch: 2, + sourceCellId: 'cell-a', + targetCellId: 'cell-b' + }) + await expect(store.completeEvacuationFromDeadSource(identity, input)).resolves.toEqual({ + changed: false, + assignmentEpoch: 2, + sourceCellId: 'cell-a', + targetCellId: 'cell-b' + }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 }) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({ + inProgress: 0 + }) + }) + + it('retires an inactive registered migration after its source is proven dead', async () => { + let now = 100 + const cells = [ + { + id: 'cell-a', + url: 'https://relay-a.example.com', + capacityRequests: 10, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 + }, + { + id: 'cell-b', + url: 'https://relay-b.example.com', + capacityRequests: 10, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 + } + ] + const store = await setupWithHeartbeats(() => now, cells) + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[1]!) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + now += 45_001 + await heartbeat(store, cells[1]!, { startedAt: 50 }) + await store.attestCellFence('cell-a', '11111111-1111-4111-8111-111111111111') + + await expect( + store.cellEvacuationStatus('cell-a', 'cell-b', true) + ).resolves.toMatchObject({ inProgress: 0, completed: 1, blocked: 0 }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 }) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([]) + expect( + await database!.query( + `SELECT state FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ? AND cell_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, 'cell-b', migration.assignmentEpoch] + ) + ).toEqual([{ state: 'released' }]) + expect( + await database!.query( + `SELECT cell_id, assignment_epoch, reserved_controls, migration_leases + FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { + cell_id: 'cell-b', + assignment_epoch: 2, + reserved_controls: 0, + migration_leases: 0 + } + ]) + }) + + it.each(['expired', 'previous-incarnation'] as const)( + 'retires a registered migration and its %s target control after the source dies', + async (targetControlState) => { + let now = 100 + const cells = [ + { + id: 'cell-a', + url: 'https://relay-a.example.com', + capacityRequests: 10, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 + }, + { + id: 'cell-b', + url: 'https://relay-b.example.com', + capacityRequests: 10, + connectionHardCap: 600 as const, + connectionUnobservedBound: 50 + } + ] + const store = await setupWithHeartbeats(() => now, cells) + await heartbeat(store, cells[0]!) + await heartbeat(store, cells[1]!) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + now += + targetControlState === 'expired' + ? ASSIGNMENT_LIMITS.activityLeaseMs + 1 + : 45_001 + await heartbeat(store, cells[1]!, { + startedAt: targetControlState === 'previous-incarnation' ? now - 1 : 50, + incarnation: + targetControlState === 'previous-incarnation' + ? '22222222-2222-4222-8222-222222222222' + : '11111111-1111-4111-8111-111111111111' + }) + await store.attestCellFence( + 'cell-a', + '11111111-1111-4111-8111-111111111111' + ) + + await expect( + store.cellEvacuationStatus('cell-a', 'cell-b', false) + ).resolves.toMatchObject({ + inProgress: 1, + registeredCompletable: 0, + registeredTargetInactive: 1 + }) + await expect( + store.cellEvacuationStatus('cell-a', 'cell-b', true) + ).resolves.toMatchObject({ inProgress: 0, completed: 1, blocked: 0 }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 }) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([]) + expect( + await database!.query( + `SELECT cell_id, assignment_epoch, reserved_controls, migration_leases + FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { + cell_id: 'cell-b', + assignment_epoch: migration.assignmentEpoch, + reserved_controls: 0, + migration_leases: 0 + } + ]) + } + ) + + it('refuses dead-source completion while the source heartbeat is fresh', async () => { + const store = await setupWithHeartbeats(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + await heartbeat(store, { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }) + await heartbeat(store, { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + + await expect( + store.attestCellFence('cell-a', '11111111-1111-4111-8111-111111111111') + ).rejects.toThrow('cell_fence_runtime_not_stale') + await expect( + store.completeEvacuationFromDeadSource(identity, { + assignmentEpoch: migration.assignmentEpoch, + sourceCellId: 'cell-a', + targetCellId: 'cell-b' + }) + ).rejects.toThrow('cell_fence_attestation_missing') + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 2 }) + }) + + it('supersedes a registered migration after its disabled target is proven unavailable', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + for (const cell of cells) await heartbeat(store, cell) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await database!.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`, + [ + 'superseded-reservation', + 'superseded-reservation', + identity.userId, + identity.relayHostId, + migration.assignmentEpoch, + 'cell-b', + 100, + 100, + 100 + ] + ) + await store.setCellEnabled('cell-b', false) + now += 45_001 + await heartbeat(store, cells[0]!, { startedAt: 50 }) + await heartbeat(store, cells[2]!, { startedAt: 50 }) + await store.attestCellFence( + 'cell-b', + '11111111-1111-4111-8111-111111111111' + ) + const input = { + assignmentEpoch: migration.assignmentEpoch, + sourceCellId: 'cell-a', + currentTargetCellId: 'cell-b', + replacementTargetCellId: 'cell-c' + } + + const replacement = await store.supersedeRegisteredEvacuation(identity, input) + expect(replacement).toMatchObject({ + sourceCellId: 'cell-a', + targetCellId: 'cell-c', + previousEpoch: 2, + assignmentEpoch: 3 + }) + await expect(store.supersedeRegisteredEvacuation(identity, input)).resolves.toEqual(replacement) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-c', assignmentEpoch: 3 }) + expect( + await database!.query( + `SELECT state FROM relay_control_connection_reservations + WHERE reservation_id = ?`, + ['superseded-reservation'] + ) + ).toEqual([{ state: 'released' }]) + expect(await cellReservations(database!)).toEqual({ + 'cell-a': 1, + 'cell-b': 0, + 'cell-c': 2 + }) + expect( + await database!.query( + `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? ORDER BY cell_id, activity_id`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { activity_kind: 'control', cell_id: 'cell-a' }, + { activity_kind: 'control', cell_id: 'cell-c' }, + { activity_kind: 'migration', cell_id: 'cell-c' } + ]) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({ + inProgress: 0 + }) + expect(await store.cellEvacuationStatus('cell-a', 'cell-c', false)).toMatchObject({ + inProgress: 1 + }) + }) + + it('retries registered migration supersession with inventory locked first', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const delegate = await openInMemoryRelayDatabase() + const retryDatabase = new OneShotInventoryFailureDatabase(delegate) + database = retryDatabase + const store = new RelayAssignmentStore(retryDatabase, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells(cells) + for (const cell of cells) await heartbeat(store, cell) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.setCellEnabled('cell-b', false) + now += 45_001 + await heartbeat(store, cells[0]!, { startedAt: 50 }) + await heartbeat(store, cells[2]!, { startedAt: 50 }) + await store.attestCellFence( + 'cell-b', + '11111111-1111-4111-8111-111111111111' + ) + retryDatabase.arm() + + await expect( + store.supersedeRegisteredEvacuation(identity, { + assignmentEpoch: migration.assignmentEpoch, + sourceCellId: 'cell-a', + currentTargetCellId: 'cell-b', + replacementTargetCellId: 'cell-c' + }) + ).resolves.toMatchObject({ targetCellId: 'cell-c', assignmentEpoch: 3 }) + expect(retryDatabase.retryLocks).toEqual([ + 'inventory-nowait-failed', + 'inventory-first', + 'assignment-nowait' + ]) + }) + + it('reconciles durable cell accounting once before aggregate supersession', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + for (const cell of cells) await heartbeat(store, cell) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.setCellEnabled('cell-b', false) + now += 45_001 + await heartbeat(store, cells[0]!, { startedAt: 50 }) + await heartbeat(store, cells[2]!, { startedAt: 50 }) + await store.attestCellFence( + 'cell-b', + '11111111-1111-4111-8111-111111111111' + ) + await database!.query( + `UPDATE relay_cells + SET reserved_requests = CASE WHEN cell_id = ? THEN 1 ELSE 0 END + WHERE cell_id IN (?, ?)`, + ['cell-c', 'cell-a', 'cell-c'] + ) + + await expect( + store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100) + ).resolves.toBe(1) + expect(await cellReservations(database!)).toEqual({ + 'cell-a': 1, + 'cell-b': 0, + 'cell-c': 2 + }) + expect( + await database!.query( + `SELECT assignment_epoch, target_cell_id, aborted_at + FROM relay_assignment_migrations + WHERE user_id = ? ORDER BY assignment_epoch`, + [identity.userId] + ) + ).toEqual([ + { assignment_epoch: 2, target_cell_id: 'cell-b', aborted_at: 45_101 }, + { assignment_epoch: 3, target_cell_id: 'cell-c', aborted_at: null } + ]) + }) + + it('preserves healthy third-cell activity during aggregate supersession', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }, + { id: 'cell-d', url: 'https://relay-d.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + for (const cell of cells) await heartbeat(store, cell) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await database!.query( + `UPDATE relay_assignment_activity_leases SET cell_id = ? + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + ['cell-d', identity.userId, identity.relayHostId, sourceControl] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = ? WHERE cell_id = ?`, + [5, 'cell-c'] + ) + await store.setCellEnabled('cell-b', false) + now += 45_001 + await heartbeat(store, cells[0]!, { startedAt: 50 }) + await heartbeat(store, cells[2]!, { startedAt: 50 }) + await heartbeat(store, cells[3]!, { startedAt: 50 }) + await store.attestCellFence( + 'cell-b', + '11111111-1111-4111-8111-111111111111' + ) + + await expect( + store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100) + ).resolves.toBe(1) + expect(await cellReservations(database!)).toEqual({ + 'cell-a': 0, + 'cell-b': 0, + 'cell-c': 1, + 'cell-d': 1 + }) + expect( + await database!.query( + `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? ORDER BY cell_id, activity_id`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { activity_kind: 'control', cell_id: 'cell-c' }, + { activity_kind: 'migration', cell_id: 'cell-c' }, + { activity_kind: 'control', cell_id: 'cell-d' } + ]) + }) + + it('refuses supersession while the registered target remains available', async () => { + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => 100, cells) + for (const cell of cells) await heartbeat(store, cell) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.setCellEnabled('cell-b', false) + + await expect( + store.attestCellFence('cell-b', '11111111-1111-4111-8111-111111111111') + ).rejects.toThrow('cell_fence_runtime_not_stale') + await expect( + store.supersedeRegisteredEvacuation(identity, { + assignmentEpoch: migration.assignmentEpoch, + sourceCellId: 'cell-a', + currentTargetCellId: 'cell-b', + replacementTargetCellId: 'cell-c' + }) + ).rejects.toThrow('cell_fence_attestation_missing') + expect( + await database!.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ cell_id: 'cell-b', assignment_epoch: 2 }]) + }) + + it('serializes concurrent retries of registered migration supersession', async () => { + let now = 100 + const cells = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 } + ] + const store = await setupWithHeartbeats(() => now, cells) + for (const cell of cells) await heartbeat(store, cell) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled('cell-a', false) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.setCellEnabled('cell-b', false) + now += 45_001 + await heartbeat(store, cells[0]!, { startedAt: 50 }) + await heartbeat(store, cells[2]!, { startedAt: 50 }) + await store.attestCellFence( + 'cell-b', + '11111111-1111-4111-8111-111111111111' + ) + const input = { + assignmentEpoch: migration.assignmentEpoch, + sourceCellId: 'cell-a', + currentTargetCellId: 'cell-b', + replacementTargetCellId: 'cell-c' + } + + const results = await Promise.all([ + store.supersedeRegisteredEvacuation(identity, input), + store.supersedeRegisteredEvacuation(identity, input) + ]) + expect(results[0]).toEqual(results[1]) + expect( + await database!.query( + `SELECT assignment_epoch FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? ORDER BY assignment_epoch`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ assignment_epoch: 2 }, { assignment_epoch: 3 }]) + expect(await cellReservations(database!)).toEqual({ + 'cell-a': 1, + 'cell-b': 0, + 'cell-c': 2 + }) + }) + + it('classifies an expired migration blocked by a newer target assignment', async () => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: 2, + generation: 1 + }) + await database!.query( + `UPDATE relay_assignments SET assignment_epoch = ? + WHERE user_id = ? AND relay_host_id = ?`, + [3, identity.userId, identity.relayHostId] + ) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({ + inProgress: 1, + oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs, + oldestRemainingMs: -1, + targetRegistered: 0, + ...NO_REGISTERED_EVACUATION_DIAGNOSTICS, + completed: 0, + blocked: 0, + expiredUnregistered: 1, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 1, + blockedExpiredOnNewerTargetAssignment: 1 + }) + }) + + it('does not complete a registered migration after the assignment epoch advances', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + await database!.query( + `UPDATE relay_assignments SET assignment_epoch = ? + WHERE user_id = ? AND relay_host_id = ?`, + [migration.assignmentEpoch + 1, identity.userId, identity.relayHostId] + ) + + expect(await store.completeReadyEvacuations()).toBe(0) + await expect( + store.completeEvacuation(identity, migration.assignmentEpoch) + ).rejects.toThrow('migration_assignment_mismatch') + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({ + inProgress: 1, + targetRegistered: 1 + }) + }) + + it('retires an expired migration without rewriting its newer target assignment', async () => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: 2, + generation: 1 + }) + await database!.query( + `UPDATE relay_assignments SET assignment_epoch = ? + WHERE user_id = ? AND relay_host_id = ?`, + [3, identity.userId, identity.relayHostId] + ) + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + expect(await store.abortExpiredEvacuations()).toBe(1) + expect(await store.resolve(identity)).toMatchObject({ + cellId: 'cell-b', + assignmentEpoch: 3 + }) + expect( + await database!.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { activity_id: 'control:cell-a:1' }, + { activity_id: 'control:cell-b:1' } + ]) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({ + inProgress: 0 + }) + }) + + it.each([ + { assignmentEpoch: 1, removeAssignment: false, expected: 'migration_assignment_mismatch' }, + { assignmentEpoch: 2, removeAssignment: true, expected: 'migration_assignment_missing' } + ])( + 'fails closed when expired migration assignment state is not superseding: $expected', + async ({ assignmentEpoch, removeAssignment, expected }) => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await store.startEvacuation(identity, 'cell-b') + if (removeAssignment) { + await database!.query( + `DELETE FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + } else { + await database!.query( + `UPDATE relay_assignments SET assignment_epoch = ? + WHERE user_id = ? AND relay_host_id = ?`, + [assignmentEpoch, identity.userId, identity.relayHostId] + ) + } + + now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + await expect(store.abortExpiredEvacuations()).rejects.toThrow(expected) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({ + inProgress: 1, + targetRegistered: 0 + }) + } + ) + + it('reconciles duplicated assignment counters and cell reservations after evacuation', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: 2, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: 2 + }) + await store.releaseActivity(identity, sourceControl) + await database!.query( + `UPDATE relay_assignments SET reserved_controls = 9, reserved_splices = 4, + reserved_invites = 3, pending_installs = 2, pending_confirmations = 2, + migration_leases = 7 WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = + CASE WHEN cell_id = ? THEN 11 ELSE 3 END`, + ['cell-a'] + ) + + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({ + inProgress: 0, + ...NO_ACTIVE_MIGRATION_LEASE, + targetRegistered: 0, + ...NO_REGISTERED_EVACUATION_DIAGNOSTICS, + completed: 1, + blocked: 0, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 }) + expect( + await database!.query( + `SELECT reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases + FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([ + { + reserved_controls: 1, + reserved_splices: 0, + reserved_invites: 0, + pending_installs: 0, + pending_confirmations: 0, + migration_leases: 0 + } + ]) + }) + + it('refuses reservation reconciliation when an activity lease has no assignment', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ['orphan-user', 'orphanhost000001', 'control:orphan', 'control', 'cell-a', 1, 200, 100] + ) + + await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow( + 'activity_lease_assignment_missing' + ) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 }) + }) + + it('refuses reservation reconciliation when an activity lease names no cell', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + await store.assign(identity) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + 'splice:missing-cell', + 'splice', + 'missing-cell', + 2, + 200, + 100 + ] + ) + + await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow( + 'activity_lease_cell_missing' + ) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 }) + }) + + it('leaves accounting for cells outside the selected evacuation pair unchanged', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 20 } + ]) + await database!.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ['unrelated-user', 'unrelatedhost001', 'cell-c', 1, 200, 100, 9, 0, 0, 0, 0, 0] + ) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + 'unrelated-user', + 'unrelatedhost001', + 'control:unrelated', + 'control', + 'cell-c', + 1, + 200, + 100 + ] + ) + await database!.query( + `UPDATE relay_cells SET reserved_requests = 9 WHERE cell_id = ?`, + ['cell-c'] + ) + + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({ + inProgress: 0 + }) + expect(await cellReservations(database!)).toEqual({ + 'cell-a': 0, + 'cell-b': 0, + 'cell-c': 9 + }) + expect( + await database!.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + ['unrelated-user', 'unrelatedhost001'] + ) + ).toEqual([{ reserved_controls: 9 }]) + }) + + it('rebalances only a fully inactive assignment after the dormant TTL', async () => { + let now = 100 + const store = await setup(() => now, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + const control = await store.activateControl(identity, { + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch, + generation: 1 + }) + await expect(store.rebalanceDormant(identity, 'cell-b')).rejects.toThrow('assignment_active') + await store.releaseActivity(identity, control) + now += ASSIGNMENT_LIMITS.dormantTtlMs + 1 + + await expect(store.rebalanceDormant(identity, 'cell-b')).resolves.toMatchObject({ + cellId: 'cell-b', + assignmentEpoch: 2 + }) + expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 }) + }) + + it('durably removes an unhealthy cell from new assignment without moving existing hosts', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + const existing = { userId: 'user-a', relayHostId: 'host000000000001' } + expect(await store.assign(existing)).toMatchObject({ cellId: 'cell-a' }) + await store.setCellEnabled('cell-a', false) + await store.reconcileCells([ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } + ]) + expect(await store.resolve(existing)).toMatchObject({ cellId: 'cell-a' }) + await expect( + store.assign({ userId: 'user-b', relayHostId: 'host000000000002' }) + ).resolves.toMatchObject({ cellId: 'cell-b' }) + await store.setCellEnabled('cell-a', true) + await expect(store.setCellEnabled('missing', false)).rejects.toThrow('cell_not_found') + }) + + it('bulk-migrates active controls without exposing assignment identities', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + await store.setCellEnabled('cell-b', false) + const identities = [1, 2, 3].map((index) => ({ + userId: `user-${index}`, + relayHostId: `host${String(index).padStart(12, '0')}` + })) + const sourceControls: string[] = [] + for (const identity of identities) { + const assignment = await store.assign(identity) + sourceControls.push( + await store.activateControl(identity, { + cellId: 'cell-a', + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + ) + } + await store.setCellEnabled('cell-b', true) + expect(await store.cellEvacuationCapacity('cell-a', 'cell-b')).toEqual({ + sourceAssignments: 3, + requiredTargetUnits: 6, + availableTargetUnits: 20 + }) + expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 2)).toBe(2) + expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 2)).toBe(1) + expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 2)).toBe(0) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({ + inProgress: 3, + oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs, + oldestRemainingMs: ASSIGNMENT_LIMITS.migrationLeaseMs, + targetRegistered: 0, + ...NO_REGISTERED_EVACUATION_DIAGNOSTICS, + completed: 0, + blocked: 0, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + for (const identity of identities) { + const assignment = await store.resolve(identity) + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: assignment!.assignmentEpoch, + generation: 2 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: assignment!.assignmentEpoch + }) + } + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({ + inProgress: 3, + oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs, + oldestRemainingMs: ASSIGNMENT_LIMITS.migrationLeaseMs, + targetRegistered: 3, + registeredSourceActive: 3, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 0, + blocked: 3, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + for (let index = 0; index < identities.length; index++) { + await store.releaseActivity(identities[index]!, sourceControls[index]!) + } + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({ + inProgress: 0, + ...NO_ACTIVE_MIGRATION_LEASE, + targetRegistered: 0, + ...NO_REGISTERED_EVACUATION_DIAGNOSTICS, + completed: 3, + blocked: 0, + ...NO_EXPIRED_EVACUATION_DIAGNOSTICS + }) + }) + + it.each(['cell-b', 'cell-c'])( + 'skips a batch row that concurrently moved from the selected source to %s', + async (movedCellId) => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }, + { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: 'cell-a', + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + const startEvacuation = store.startEvacuation.bind(store) + vi.spyOn(store, 'startEvacuation').mockImplementationOnce(async (...args) => { + await database!.query( + `UPDATE relay_assignments SET cell_id = ? WHERE user_id = ? AND relay_host_id = ?`, + [movedCellId, identity.userId, identity.relayHostId] + ) + return await startEvacuation(...args) + }) + + expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 10)).toBe(0) + expect(await store.resolve(identity)).toMatchObject({ cellId: movedCellId }) + expect(await database!.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) + } + ) + + it('does not count an existing migration after a stale batch row moves to its target', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: 'cell-a', + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + const startEvacuation = store.startEvacuation.bind(store) + vi.spyOn(store, 'startEvacuation').mockImplementationOnce(async (...args) => { + await startEvacuation(identity, 'cell-b') + return await startEvacuation(...args) + }) + + expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 10)).toBe(0) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b' }) + expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({ + inProgress: 1 + }) + }) + + it('preserves operator-owned tagged URLs and admission state across reconciliation', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 } + ]) + await store.configureCell( + { id: 'cell-a', url: 'https://old---relay-a.example.com', capacityRequests: 20 }, + false + ) + await store.reconcileCells([ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 } + ]) + const rows = await database!.query( + `SELECT cell_url, enabled, capacity_requests FROM relay_cells WHERE cell_id = ?`, + ['cell-a'] + ) + expect(rows).toEqual([ + { + cell_url: 'https://old---relay-a.example.com', + enabled: 0, + capacity_requests: 20 + } + ]) + }) + + it('bulk-migrates activity even when its source control is temporarily absent', async () => { + const store = await setup(() => 100, [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 } + ]) + const identity = { userId: 'user-1', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + const control = await store.activateControl(identity, { + cellId: 'cell-a', + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.acquireActivity(identity, { + activityId: 'splice-without-control', + kind: 'splice', + cellId: 'cell-a' + }) + await store.releaseActivity(identity, control) + expect(await store.cellEvacuationCapacity('cell-a', 'cell-b')).toEqual({ + sourceAssignments: 1, + requiredTargetUnits: 3, + availableTargetUnits: 20 + }) + expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 10)).toBe(1) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 }) + }) +}) + +async function cellReservations(database: RelayDatabase): Promise> { + const rows = await database.query( + `SELECT cell_id, reserved_requests FROM relay_cells ORDER BY cell_id ASC` + ) + return Object.fromEntries(rows.map((row) => [String(row.cell_id), Number(row.reserved_requests)])) +} diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts new file mode 100644 index 00000000000..0b2b1ef72a9 --- /dev/null +++ b/cloud/apps/relay/src/assignment-store.ts @@ -0,0 +1,8347 @@ +import { randomUUID } from 'node:crypto' +import { performance } from 'node:perf_hooks' +import { + ASSIGNMENT_LIMITS, + mayNormallyReassign, + RELAY_ADMISSION_BUDGETS, + RELAY_DEFAULT_REGION, + RELAY_REGIONS, + RELAY_PROTOCOL_LIMITS, + type RelayRegion +} from '@orca-cloud/relay-contract' +import { + cellAdmissionState, + cellAdmissionStates, + ensureCellAdmission, + parseCellAdmissionState, + RelayCellAdmissionSelector, + setCellAdmissionBeforeBoundary, + stateFromEnabled, + synchronizeCellAdmissionBoundary, + type CellAdmissionMembership, + type CellAdmissionSelectorInspection, + type CellAdmissionState +} from './cell-admission-selector.js' +import { + RelayMigrationCellRegistrar, + type MigrationCellRegistration +} from './cell-admission-migration-registration.js' +import { + ASSIGNMENT_CONNECTION_HEADROOM_QUERY +} from './assignment-connection-headroom-query.js' +import { AssignmentIdentityQueue } from './assignment-identity-queue.js' +import type { RelayCellConfig } from './config.js' +import type { RelayDatabase, RelayTransactionOptions, SqlRow } from './database.js' +import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' +import { + combineRegionalRehomeSafety, + REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT, + REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT, + regionalRehomePoolPressure, + regionalRehomeSafetyFailure +} from './regional-rehome-safety.js' +import { + ABANDONED_REGISTERED_MIGRATION, + DURABLY_FENCED_MIGRATION_SOURCE, + REGISTERED_MIGRATION_ABANDON_MS +} from './registered-migration-abandonment.js' + +type AssignmentIdentity = { userId: string; relayHostId: string } +type CellHeartbeat = { + cellId: string + cellUrl: string + cellIncarnation: string + startedAt: number + ready: boolean + observedRequests: number + region?: RelayRegion + totalConnections?: number + inFlightConnections?: number + reservedConnectionUnits?: number + enforcedConnectionUnits?: number + connectionInclusionWatermark?: number + connectionHardCap?: number + connectionUnobservedBound?: number +} + +type CellRegionalRehomeStatus = { + cellId: string + cellIncarnation: string + regionalRehomeProtocol: number + safety: RegionalRehomeSafetySnapshot +} + +type RelayAssignmentStoreOptions = { + requireLiveCells?: boolean + heartbeatTtlMs?: number + recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void +} + +export type ControlRenewalOutcome = + | 'renewed' + | 'assignment_not_found' + | 'activity_cell_not_authoritative' + | 'control_activity_not_found' + | 'control_activity_moved' + | 'database_error' + +const CONTROL_RENEWAL_OUTCOMES = new Set([ + 'renewed', + 'assignment_not_found', + 'activity_cell_not_authoritative', + 'control_activity_not_found', + 'control_activity_moved' +]) +export type RelayAssignment = AssignmentIdentity & { + cellId: string + cellUrl: string + assignmentEpoch: number + leaseExpiresAt: number + region?: RelayRegion +} + +export type RelayRegionCatalogEntry = { + region: RelayRegion + probeOrigins: string[] +} + +export type RelayAssignmentMigration = AssignmentIdentity & { + sourceCellId: string + targetCellId: string + previousEpoch: number + assignmentEpoch: number + expiresAt: number + targetRegisteredAt?: number +} + +export type RegionalRehomeAttempt = AssignmentIdentity & { + attemptId: string + preferredRegion: 'asia-east2' + sourceCellId: string + sourceCellUrl: string + sourceCellIncarnation: string + targetCellId: string + targetCellIncarnation: string + previousEpoch: number + assignmentEpoch: number + drainGraceMs: number + sendAttempts: number +} + +export type RegionalHostDrainOutcome = + | 'accepted' + | 'already-accepted' + | 'host-not-connected' + +export type RegionalRehomeFleetSafety = RegionalRehomeSafetySnapshot & { + requiredCells: number + missingCells: number + maxReconnects: number +} + +export type RegionalRehomeControl = { + generation: number + enabled: boolean + observationStartedAt: number + notBefore: number + ratePerMinute: number + preferenceMaxAgeMs: number + drainGraceMs: number +} + +type RegisteredEvacuationSupersessionInput = { + assignmentEpoch: number + sourceCellId: string + currentTargetCellId: string + replacementTargetCellId: string +} + +export type DeadSourceCompletionResult = { + changed: boolean + assignmentEpoch: number + sourceCellId: string + targetCellId: string +} + +export type CellEvacuationStatus = { + inProgress: number + oldestExpiresAt: number | null + oldestRemainingMs: number | null + targetRegistered: number + registeredSourceActive: number + registeredCompletable: number + registeredTargetInactive: number + completed: number + blocked: number + expiredUnregistered: number + repairableExpiredUnregistered: number + abortableExpiredUnregistered: number + blockedExpiredUnregistered: number + blockedExpiredOnNewerTargetAssignment: number +} + +export type CellEvacuationCapacity = { + sourceAssignments: number + requiredTargetUnits: number + availableTargetUnits: number +} + +export type CellDeploymentStatus = { + cellId: string + cellUrl: string + region: RelayRegion + enabled: boolean + admissionState: CellAdmissionState + capacityRequests: number + reservedRequests: number + assignments: number + activityLeases: number + activityRequestUnits: number + restartBlockingActivityLeases: number + restartBlockingActivityRequestUnits: number + restartBlockingReservedRequests: number + outgoingMigrations: number + incomingMigrations: number + connectionCapacity: null | { + hardCap: number + controlRebindReserve: number + ordinaryConnectionLimit: number + unobservedBound: number + normalAdmissionPause: number + observedConnections: number + inFlightConnections: number + reservedConnectionUnits: number + enforcedConnectionUnits: number + pendingControlReservations: number + heartbeatFresh: boolean + } + runtime: null | { + cellUrl: string + cellIncarnation: string + startedAt: number + ready: boolean + observedRequests: number + lastHeartbeatAt: number + heartbeatFresh: boolean + regionalRehomeProtocol: number + } +} + +export type CellFenceAttemptEvidence = { + attemptId: string + environment: 'staging' | 'production' + cellId: string + cellIncarnation: string + migName: string + instanceGroup: string + generationIdentity: string + fenceCommit: string + planSha256: string + planObjectName: string + planObjectGeneration?: string + varFileSha256: string + terraformStateLineage: string + terraformStateSerial: number + terraformStateObjectGeneration: string + terraformStateObjectSha256: string + requestReason: string +} + +export type CellFenceApplyInvocation = { + invocationId: string + requestReason: string + startedAt: number + gceOperation?: string +} + +export type CellFenceAttempt = CellFenceAttemptEvidence & { + applyInvocations?: CellFenceApplyInvocation[] + gceOperation?: string + createdAt: number + expiresAt: number + applyStartedAt?: number + completedAt?: number + abortedAt?: number +} + +export type CellDrainAttemptState = + | 'prepared' + | 'send-may-have-started' + | 'application-receipt' + | 'proven-not-delivered' + +export type CellDrainAttempt = { + attemptId: string + cellId: string + cellIncarnation: string + traceValue: string + plannedGraceMs: number + state: CellDrainAttemptState + preparedAt: number + sendMayHaveStartedAt?: number + sendPermitExpiresAt?: number + applicationReceiptAt?: number + backendSuccessStatus?: number + backendInstance?: string + receiptCellIncarnation?: string + retryAfter?: number + recoverForwardAttemptedAt?: number + provenNotDeliveredAt?: number +} + +export type AssignmentActivityKind = + | 'control' + | 'splice' + | 'invite' + | 'install' + | 'confirmation' + | 'migration' + +const ACTIVITY_COLUMN: Record = { + control: 'reserved_controls', + splice: 'reserved_splices', + invite: 'reserved_invites', + install: 'pending_installs', + confirmation: 'pending_confirmations', + migration: 'migration_leases' +} + +const ACTIVITY_REQUEST_UNITS: Record = { + control: 1, + splice: 2, + invite: 1, + install: 1, + confirmation: 1, + migration: 1 +} + +const ASSIGNMENT_LOCK_RETRY_DEADLINE_MS = 15_000 +// Why: stranded detection (issue #225) needs a grant old enough that a real +// attach would have registered (the 90s activity lease covers dial + +// activation), yet recent enough to prove an active retry loop rather than +// ordinary dormancy — which stays governed by the 24h rule. +const STRANDED_MIN_GRANT_AGE_MS = 60_000 +const STRANDED_RECENT_ACTIVITY_MS = 15 * 60_000 +const REGION_PREFERENCE_RETENTION_MS = 30 * 24 * 60 * 60_000 +const REGIONAL_REHOME_UNREGISTERED_REFRESH_MS = 5 * 60_000 +const REGIONAL_REHOME_MAX_REFRESH_MS = 24 * 60 * 60_000 +// Drain grace is enforced by session-scoped cell state that any control +// reconnect sheds, so receipted attempts can stall dual-homed past grace; +// redrains re-dispatch them with the elapsed (zero) grace until they detach. +export const REGIONAL_REHOME_REDRAIN_INTERVAL_MS = 60_000 +export const REGIONAL_REHOME_REDRAIN_SEND_LIMIT = 20 +export const REGIONAL_REHOME_QUARANTINE_FAILURES = 3 +export const REGIONAL_REHOME_QUARANTINE_MS = 15 * 60_000 +const REGIONAL_REHOME_QUARANTINE_EXCLUSION_LIMIT = 50 +const REGIONAL_REHOME_QUARANTINE_MEMORY_LIMIT = 1_000 +const REGIONAL_REHOME_OBSERVATION_MS = 24 * 60 * 60_000 +const ASSIGNMENT_LOCK_RETRY_MAX_DELAY_MS = 50 +type AssignmentInventoryScope = 'none' | 'general' | 'all' +type RetriedAssignmentInventoryScope = Exclude + +class AssignmentInventoryLockUnavailable extends Error { + constructor(readonly inventoryScope: RetriedAssignmentInventoryScope) { + super('database_lock_unavailable') + } +} + +class AssignmentInventoryScopeChanged extends Error { + constructor() { + super('assignment_inventory_scope_changed') + } +} + +// Debt holds connection headroom for a control that may still arrive shortly +// after its director-side timeout. Nothing legitimately arrives minutes late +// (attach deadline 10s, orphan grace 30s); unretired debt from hosts that +// never return otherwise starves connection headroom fleet-wide and turns +// every placement into relay_capacity_exhausted. +const LATE_ARRIVAL_DEBT_RETENTION_MS = 10 * 60 * 1_000 +const CELL_FENCE_TTL_MS = 5 * 60 * 1_000 +const CELL_FENCE_ATTEMPT_TTL_MS = 60 * 60 * 1_000 +const CELL_DRAIN_SEND_PERMIT_MS = 30_000 +const MAX_DRAIN_ACCOUNTING_REPAIR_ATTEMPTS = 3 +const CELL_FENCE_ATTEMPT_SELECT = ` + SELECT attempts.*, bindings.plan_object_name, bindings.plan_object_generation, + bindings.var_file_sha256, bindings.terraform_state_lineage, + bindings.terraform_state_serial, bindings.terraform_state_object_generation, + bindings.terraform_state_object_sha256, bindings.request_reason + FROM relay_cell_fence_attempts attempts + JOIN relay_cell_fence_plan_bindings bindings + ON bindings.attempt_id = attempts.attempt_id` +export const STRANDED_MIGRATION_ABANDON_MS = REGISTERED_MIGRATION_ABANDON_MS +const ABORTABLE_EXPIRED_MIGRATION = `( + ( + migration.target_registered_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM relay_post_drain_migration_pins pin + WHERE pin.user_id = migration.user_id + AND pin.relay_host_id = migration.relay_host_id + AND pin.assignment_epoch = migration.assignment_epoch + ) + ) + OR ${ABANDONED_REGISTERED_MIGRATION} +)` + +export class RelayAssignmentStore { + private readonly requireLiveCells: boolean + private readonly heartbeatTtlMs: number + // Poisoned attempts never complete or abort and stay the oldest rows, so + // unquarantined they eventually fill the sweeps' LIMIT page and starve + // every healthy candidate. Process-local on purpose: it resets on deploy + // and each director relearns within a few ticks. + private readonly regionalRehomeCandidateQuarantine = new Map< + string, + { failures: number; until: number } + >() + private readonly recordControlRenewal?: RelayAssignmentStoreOptions['recordControlRenewal'] + private readonly admissionSelector: RelayCellAdmissionSelector + private readonly migrationCellRegistrar: RelayMigrationCellRegistrar + private readonly activityQueue = new AssignmentIdentityQueue() + private assignmentTail: Promise = Promise.resolve() + private pendingRegionalRehomeDisableLog: Record | null = null + + constructor( + private readonly database: RelayDatabase, + private readonly now: () => number = Date.now, + options: RelayAssignmentStoreOptions = {} + ) { + this.requireLiveCells = options.requireLiveCells ?? false + this.heartbeatTtlMs = options.heartbeatTtlMs ?? 45_000 + this.recordControlRenewal = options.recordControlRenewal + this.admissionSelector = new RelayCellAdmissionSelector(database, now) + this.migrationCellRegistrar = new RelayMigrationCellRegistrar(database, now) + } + + async reconcileCells(cells: RelayCellConfig[], disableMissing = true): Promise { + await this.reconcileCellsWithOptions(cells, disableMissing) + } + + async reconcileCellsAtStartup(cells: RelayCellConfig[]): Promise { + await this.reconcileCellsWithOptions(cells, false, { reportRetries: false }) + } + + private async reconcileCellsWithOptions( + cells: RelayCellConfig[], + disableMissing: boolean, + transactionOptions?: RelayTransactionOptions + ): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`) + // Capacity rows share one global lock order with assignment transactions. + for (const cell of [...cells].sort((left, right) => left.id.localeCompare(right.id))) { + // Operator-owned enabled state and tagged URLs must survive revision restarts. + await transaction.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + capacity_requests = excluded.capacity_requests, + last_heartbeat_at = excluded.last_heartbeat_at, + updated_at = excluded.updated_at`, + [ + cell.id, + cell.url, + cell.initiallyEnabled === false ? 0 : 1, + cell.capacityRequests, + 0, + 0, + now, + now + ] + ) + await transaction.query( + `INSERT INTO relay_cell_regions (cell_id, region) VALUES (?, ?) + ON CONFLICT (cell_id) DO UPDATE SET region = excluded.region`, + [cell.id, cell.region ?? RELAY_DEFAULT_REGION] + ) + await ensureCellAdmission( + transaction, + cell.id, + stateFromEnabled(cell.initiallyEnabled !== false), + now + ) + if ( + cell.connectionHardCap !== undefined && + cell.connectionUnobservedBound !== undefined + ) { + const currentLimit = ( + await transaction.queryLocked( + `SELECT hard_cap, unobserved_bound FROM relay_cell_connection_limits + WHERE cell_id = ?`, + [cell.id] + ) + )[0] + const limitChanged = + currentLimit !== undefined && + (integer(currentLimit, 'hard_cap') !== cell.connectionHardCap || + integer(currentLimit, 'unobserved_bound') !== + cell.connectionUnobservedBound) + await transaction.query( + `INSERT INTO relay_cell_connection_limits + (cell_id, hard_cap, unobserved_bound, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + hard_cap = excluded.hard_cap, + unobserved_bound = excluded.unobserved_bound, + updated_at = excluded.updated_at`, + [ + cell.id, + cell.connectionHardCap, + cell.connectionUnobservedBound, + now + ] + ) + if (limitChanged) { + await transaction.query( + `DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, + [cell.id] + ) + await transaction.query( + `DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, + [cell.id] + ) + } + } + } + const current = await transaction.query( + `SELECT cell_id, enabled FROM relay_cells ORDER BY cell_id ASC` + ) + for (const row of current) { + await ensureCellAdmission( + transaction, + text(row, 'cell_id'), + stateFromEnabled(integer(row, 'enabled') === 1), + now + ) + } + if (disableMissing) { + const ids = new Set(cells.map(({ id }) => id)) + for (const row of current) { + const id = text(row, 'cell_id') + if ( + !ids.has(id) && + (await cellAdmissionState(transaction, id)) !== 'existing-only' + ) { + await setCellAdmissionBeforeBoundary(transaction, id, 'existing-only', now) + } + } + } + await synchronizeCellAdmissionBoundary(transaction, now) + }, transactionOptions) + } + + async assign( + identity: AssignmentIdentity, + preferredRegion?: RelayRegion, + placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION + ): Promise { + const sticky = await this.assignStickyWithLockRetry(identity, preferredRegion) + if (sticky) return sticky + // Only placement needs the global inventory critical section; queueing those + // attempts locally avoids turning true placement bursts into NOWAIT storms. + return await this.serializeAssignment( + async () => await this.assignWithLockRetry(identity, preferredRegion, placementRegion) + ) + } + + private async assignStickyWithLockRetry( + identity: AssignmentIdentity, + preferredRegion?: RelayRegion + ): Promise { + return await this.withAssignmentLockRetry( + async (inventoryFirst) => + await this.assignStickyOnce(identity, inventoryFirst, preferredRegion) + ) + } + + private async assignWithLockRetry( + identity: AssignmentIdentity, + preferredRegion?: RelayRegion, + placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION + ): Promise { + const deadline = Date.now() + ASSIGNMENT_LOCK_RETRY_DEADLINE_MS + let inventoryScope: AssignmentInventoryScope = 'none' + while (true) { + try { + return await this.assignOnce(identity, inventoryScope, preferredRegion, placementRegion) + } catch (error) { + if (error instanceof AssignmentInventoryScopeChanged) { + inventoryScope = 'all' + continue + } + if ( + !(error instanceof AssignmentInventoryLockUnavailable) || + Date.now() >= deadline + ) { + throw error + } + inventoryScope = error.inventoryScope + } + await waitForAssignmentLockRetry(deadline) + } + } + + private async withAssignmentLockRetry( + operation: (inventoryFirst: boolean) => Promise + ): Promise { + const deadline = Date.now() + ASSIGNMENT_LOCK_RETRY_DEADLINE_MS + let inventoryFirst = false + while (true) { + try { + return await operation(inventoryFirst) + } catch (error) { + if (!isDatabaseLockUnavailable(error) || Date.now() >= deadline) throw error + inventoryFirst = true + } + // The retry joins the cell queue without holding later legacy-cycle locks. + await waitForAssignmentLockRetry(deadline) + } + } + + private async assignStickyOnce( + identity: AssignmentIdentity, + inventoryFirst: boolean, + preferredRegion?: RelayRegion + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const lockedCells = inventoryFirst + ? await this.lockCellInventory(transaction) + : undefined + const existing = await this.assignmentRow(transaction, identity, inventoryFirst) + if (!existing) return null + const activityLeases = await this.lockAssignmentActivities(transaction, identity, true) + await this.recordRegionPreference(transaction, identity, preferredRegion, now) + if (mayNormallyReassign(activity(existing), now)) return null + // Why: a stranded host must fall through to placement — re-granting the + // pinned cell here is what refreshes its own activity and sustains the + // loop (issue #225). + if (await this.assignmentStrandedOnUnservedCell(transaction, identity, existing, now)) { + return null + } + + const currentCellId = text(existing, 'cell_id') + const hadControl = holdsControlLease( + activityLeases, + currentCellId, + integer(existing, 'assignment_epoch') + ) + const currentRow = lockedCells + ? lockedCells.find((row) => text(row, 'cell_id') === currentCellId) + : hadControl + ? ( + await transaction.query(`SELECT * FROM relay_cells WHERE cell_id = ?`, [ + currentCellId + ]) + )[0] + : ( + await transaction.queryLocked( + `SELECT * FROM relay_cells WHERE cell_id = ?`, + [currentCellId], + { failIfUnavailable: true } + ) + )[0] + if (!currentRow) throw new Error('assigned_cell_missing') + if ( + this.requireLiveCells && + !(await this.cellIsLive(transaction, currentCellId, now)) + ) { + return null + } + + if ( + !hadControl && + !(await this.cellHasConnectionHeadroom(transaction, currentCellId)) + ) { + if (requestUnits(existing) === 0) return null + throw new Error('relay_connection_headroom_exhausted') + } + + const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs + if (hadControl) { + await this.touchAssignment(transaction, identity, leaseExpiresAt, now) + } else { + const nextReservation = integer(currentRow, 'reserved_requests') + 1 + if (nextReservation > integer(currentRow, 'capacity_requests')) { + throw new Error('relay_capacity_exhausted') + } + await transaction.query( + `UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`, + [nextReservation, now, currentCellId] + ) + await this.adjustActivityCount(transaction, identity, 'control', 1, leaseExpiresAt, now) + await this.insertPendingControlLease( + transaction, + identity, + currentCellId, + integer(existing, 'assignment_epoch'), + now + ) + } + return this.result( + identity, + existing, + cell(currentRow, await this.cellRegion(transaction, currentCellId)), + leaseExpiresAt + ) + }) + } + + // Why: PR #194 pins assignments on existing-only cells so capacity pressure + // cannot scatter existing hosts — assuming those cells still serve them. A + // decommissioning cell (C3) broke that assumption: existing-only AND + // rejecting attaches, so pinned hosts loop forever while each grant + // refreshes their own last_activity_at, keeping dormancy-based + // reassignment permanently out of reach (issue #225). "Stranded" therefore + // requires proof the cell is not serving this host: a recent grant + // (last_activity_at inside the window) that had ample time to attach and + // still produced no live real activity. The evidence must come from the + // assignment row itself — reservation rows do not exist for cells without + // connection limits (C3), and expired pending leases are cleaned within a + // maintenance cycle, so neither reliably survives until the next grant. + private async assignmentStrandedOnUnservedCell( + transaction: RelayDatabase, + identity: AssignmentIdentity, + existing: SqlRow, + now: number + ): Promise { + const lastActivityAt = integer(existing, 'last_activity_at') + if ( + now < lastActivityAt + STRANDED_MIN_GRANT_AGE_MS || + now >= lastActivityAt + STRANDED_RECENT_ACTIVITY_MS + ) { + return false + } + const admissionRow = ( + await transaction.query( + `SELECT admission_state FROM relay_cell_admission WHERE cell_id = ?`, + [text(existing, 'cell_id')] + ) + )[0] + // Unknown or missing admission fails safe: the pin stays. + if (admissionRow?.['admission_state'] !== 'existing-only') { + return false + } + const liveLeases = ( + await transaction.query( + `SELECT COUNT(*) AS live FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND expires_at > ? + AND activity_id NOT LIKE 'control-pending:%'`, + [identity.userId, identity.relayHostId, now] + ) + )[0] + return integer(liveLeases!, 'live') === 0 + } + + private async serializeAssignment(operation: () => Promise): Promise { + const previous = this.assignmentTail + let release!: () => void + this.assignmentTail = new Promise((resolve) => (release = resolve)) + await previous + try { + return await operation() + } finally { + release() + } + } + + private async assignOnce( + identity: AssignmentIdentity, + inventoryScope: AssignmentInventoryScope, + preferredRegion?: RelayRegion, + placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION + ): Promise { + const now = this.now() + let retryScope: RetriedAssignmentInventoryScope = + inventoryScope === 'all' ? 'all' : 'general' + return await this.database.transaction(async (transaction) => { + let lockedCells = + inventoryScope === 'all' + ? await this.lockCellInventory(transaction) + : inventoryScope === 'general' + ? await this.lockGeneralCellInventory(transaction) + : undefined + const existing = await this.assignmentRow( + transaction, + identity, + inventoryScope !== 'none' + ) + retryScope = existing ? 'all' : 'general' + if (inventoryScope === 'general' && existing) { + throw new AssignmentInventoryScopeChanged() + } + const activityLeases = await this.lockAssignmentActivities(transaction, identity, true) + await this.recordRegionPreference(transaction, identity, preferredRegion, now) + let forcedDeadReassignment = false + let connectionHeadroomReassignment = false + let strandedReassignment = false + if (existing && !mayNormallyReassign(activity(existing), now)) { + lockedCells ??= await this.lockCellInventory(transaction, true) + const admission = await cellAdmissionStates(transaction) + const currentRow = lockedCells.find( + (row) => text(row, 'cell_id') === text(existing, 'cell_id') + ) + if (!currentRow) throw new Error('assigned_cell_missing') + const current = cell( + currentRow, + await this.cellRegion(transaction, text(existing, 'cell_id')) + ) + // Why: an existing-only cell that rejects attaches (C3's decommission + // posture) must not re-pin the hosts it refuses; the dead-cell fence + // path below does not apply either — the cell is live, just unwilling. + strandedReassignment = await this.assignmentStrandedOnUnservedCell( + transaction, + identity, + existing, + now + ) + if ( + !strandedReassignment && + (!this.requireLiveCells || (await this.cellIsLive(transaction, current.cellId, now))) + ) { + const hadControl = holdsControlLease( + activityLeases, + current.cellId, + integer(existing, 'assignment_epoch') + ) + const hasConnectionHeadroom = + hadControl || + (await this.cellHasConnectionHeadroom(transaction, current.cellId)) + if (hasConnectionHeadroom) { + const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs + if (hadControl) { + await this.touchAssignment(transaction, identity, leaseExpiresAt, now) + } else { + await this.adjustCellReservation(transaction, current.cellId, 1) + await this.adjustActivityCount( + transaction, + identity, + 'control', + 1, + leaseExpiresAt, + now + ) + await this.insertPendingControlLease( + transaction, + identity, + current.cellId, + integer(existing, 'assignment_epoch'), + now + ) + } + return this.result(identity, existing, current, leaseExpiresAt) + } + if (requestUnits(existing) > 0) { + throw new Error('relay_connection_headroom_exhausted') + } + if (admission.get(current.cellId) !== 'general') { + throw new Error('relay_connection_headroom_exhausted') + } + connectionHeadroomReassignment = true + } + if (!strandedReassignment && !connectionHeadroomReassignment) { + if ( + (await this.deadCellRequiresCommittedFence( + transaction, + identity, + current.cellId, + integer(existing, 'assignment_epoch') + )) && + !(await this.cellHasCommittedFence(transaction, current.cellId, now)) + ) { + throw new Error('relay_capacity_exhausted') + } + forcedDeadReassignment = true + } + } + + lockedCells ??= existing + ? await this.lockCellInventory(transaction, true) + : await this.lockGeneralCellInventory(transaction, true) + const target = await this.leastLoadedCell( + transaction, + lockedCells, + placementRegion + ) + if (!target) throw new Error('relay_capacity_exhausted') + const previousUnits = existing ? requestUnits(existing) : 0 + if (existing) { + await this.adjustCellReservation(transaction, text(existing, 'cell_id'), -previousUnits) + if (forcedDeadReassignment || strandedReassignment) { + // A fenced/dead incarnation cannot own drainable work, and a + // stranded host's only leases are the unclaimed grant artifacts of + // its own loop. Removing them prevents late expiry from + // decrementing the replacement. + await transaction.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + } + } + await this.adjustCellReservation(transaction, target.cellId, 1) + const assignmentEpoch = existing ? integer(existing, 'assignment_epoch') + 1 : 1 + const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs + await transaction.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (user_id, relay_host_id) DO UPDATE SET + cell_id = excluded.cell_id, + assignment_epoch = excluded.assignment_epoch, + lease_expires_at = excluded.lease_expires_at, + last_activity_at = excluded.last_activity_at, + reserved_controls = excluded.reserved_controls, + reserved_splices = excluded.reserved_splices, + reserved_invites = excluded.reserved_invites, + pending_installs = excluded.pending_installs, + pending_confirmations = excluded.pending_confirmations, + migration_leases = excluded.migration_leases`, + [ + identity.userId, + identity.relayHostId, + target.cellId, + assignmentEpoch, + leaseExpiresAt, + now, + 1, + 0, + 0, + 0, + 0, + 0 + ] + ) + if (existing) { + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + text(existing, 'cell_id'), + integer(existing, 'assignment_epoch'), + now + ) + } + await this.insertPendingControlLease( + transaction, + identity, + target.cellId, + assignmentEpoch, + now + ) + return { ...identity, ...target, assignmentEpoch, leaseExpiresAt } + }).catch((error: unknown) => { + if (isDatabaseLockUnavailable(error)) { + throw new AssignmentInventoryLockUnavailable(retryScope) + } + throw error + }) + } + + async resolve(identity: AssignmentIdentity): Promise { + const rows = await this.database.query( + `SELECT assignment.*, cell.cell_url, region.region + FROM relay_assignments assignment + JOIN relay_cells cell ON cell.cell_id = assignment.cell_id + LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id + WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + const row = rows[0] + if ( + row && + this.requireLiveCells && + !(await this.cellIsLive(this.database, text(row, 'cell_id'), this.now())) + ) { + return null + } + return row + ? { + ...identity, + cellId: text(row, 'cell_id'), + cellUrl: text(row, 'cell_url'), + region: optionalRelayRegion(row, 'region') ?? RELAY_DEFAULT_REGION, + assignmentEpoch: integer(row, 'assignment_epoch'), + leaseExpiresAt: integer(row, 'lease_expires_at') + } + : null + } + + async setCellEnabled(cellId: string, enabled: boolean): Promise { + await this.setCellAdmissionState(cellId, stateFromEnabled(enabled)) + } + + async setCellAdmissionState(cellId: string, state: CellAdmissionState): Promise { + await this.database.transaction( + async (transaction) => + await setCellAdmissionBeforeBoundary(transaction, cellId, state, this.now()) + ) + } + + async applyCellAdmissionSelector(input: { + attemptId: string + expectedGeneration: number + expectedMembershipSha256?: string + membership: CellAdmissionMembership + }): Promise<{ + changed: boolean + selector: { + generation: number + attemptId: string | null + membership: CellAdmissionMembership + } + }> { + return await this.admissionSelector.apply(input) + } + + async inspectCellAdmissionSelector( + attemptId?: string + ): Promise { + return await this.admissionSelector.inspect(attemptId) + } + + async addMigrationCells(input: { + attemptId: string + expectedGeneration: number + cells: MigrationCellRegistration[] + }): Promise<{ + changed: boolean + selector: { + generation: number + attemptId: string | null + membership: CellAdmissionMembership + } + }> { + return await this.migrationCellRegistrar.add(input) + } + + async recordCellHeartbeat(input: CellHeartbeat): Promise { + const now = this.now() + let inclusionWatermark: number | undefined + await this.database.transaction(async (transaction) => { + const configured = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [input.cellId]) + )[0] + if (!configured) throw new Error('cell_not_found') + if (text(configured, 'cell_url') !== input.cellUrl) throw new Error('cell_origin_mismatch') + if ( + (await this.cellRegion(transaction, input.cellId)) !== + (input.region ?? RELAY_DEFAULT_REGION) + ) { + throw new Error('cell_region_mismatch') + } + const current = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + input.cellId + ]) + )[0] + if ( + current && + ((text(current, 'cell_incarnation') === input.cellIncarnation && + input.startedAt !== integer(current, 'started_at')) || + (text(current, 'cell_incarnation') !== input.cellIncarnation && + input.startedAt <= integer(current, 'started_at'))) + ) { + throw new Error('stale_cell_incarnation') + } + const connectionLimit = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_connection_limits WHERE cell_id = ?`, + [input.cellId] + ) + )[0] + if (connectionLimit) { + if ( + input.totalConnections === undefined || + input.inFlightConnections === undefined || + input.reservedConnectionUnits === undefined || + input.enforcedConnectionUnits === undefined || + input.enforcedConnectionUnits !== + input.totalConnections + + input.inFlightConnections + + input.reservedConnectionUnits || + input.connectionHardCap !== integer(connectionLimit, 'hard_cap') || + input.connectionUnobservedBound !== + integer(connectionLimit, 'unobserved_bound') + ) { + throw new Error('cell_connection_telemetry_mismatch') + } + } else if ( + input.totalConnections !== undefined || + input.inFlightConnections !== undefined || + input.reservedConnectionUnits !== undefined || + input.enforcedConnectionUnits !== undefined || + input.connectionInclusionWatermark !== undefined || + input.connectionHardCap !== undefined || + input.connectionUnobservedBound !== undefined + ) { + throw new Error('cell_connection_limit_not_configured') + } + await transaction.query( + `INSERT INTO relay_cell_runtime + (cell_id, cell_url, cell_incarnation, started_at, ready, + observed_requests, last_heartbeat_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_url = excluded.cell_url, + cell_incarnation = excluded.cell_incarnation, + started_at = excluded.started_at, + ready = excluded.ready, + observed_requests = excluded.observed_requests, + last_heartbeat_at = excluded.last_heartbeat_at, + updated_at = excluded.updated_at`, + [ + input.cellId, + input.cellUrl, + input.cellIncarnation, + input.startedAt, + input.ready ? 1 : 0, + input.observedRequests, + now, + now + ] + ) + // Live directors read relay_cell_runtime; keep heartbeats off the placement capacity lock. + if (!this.requireLiveCells) { + await transaction.query( + `UPDATE relay_cells SET observed_requests = ?, last_heartbeat_at = ?, updated_at = ? + WHERE cell_id = ?`, + [input.observedRequests, now, now, input.cellId] + ) + } + if (connectionLimit) { + const currentSnapshot = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_connection_snapshots WHERE cell_id = ?`, + [input.cellId] + ) + )[0] + const previousWatermark = + currentSnapshot && + text(currentSnapshot, 'cell_incarnation') === input.cellIncarnation + ? integer(currentSnapshot, 'inclusion_watermark') + : -1 + inclusionWatermark = input.connectionInclusionWatermark ?? previousWatermark + 1 + if ( + input.connectionInclusionWatermark !== undefined && + inclusionWatermark <= previousWatermark + ) { + throw new Error('stale_connection_snapshot') + } + await transaction.query( + `INSERT INTO relay_cell_connection_runtime + (cell_id, cell_incarnation, total_connections, in_flight_connections, + reserved_connection_units, enforced_connection_units, last_heartbeat_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + total_connections = excluded.total_connections, + in_flight_connections = excluded.in_flight_connections, + reserved_connection_units = excluded.reserved_connection_units, + enforced_connection_units = excluded.enforced_connection_units, + last_heartbeat_at = excluded.last_heartbeat_at, + updated_at = excluded.updated_at`, + [ + input.cellId, + input.cellIncarnation, + input.totalConnections, + input.inFlightConnections, + input.reservedConnectionUnits, + input.enforcedConnectionUnits, + now, + now + ] + ) + await transaction.query( + `INSERT INTO relay_cell_connection_snapshots + (cell_id, cell_incarnation, inclusion_watermark, total_connections, + in_flight_connections, reserved_connection_units, + enforced_connection_units, snapshot_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + inclusion_watermark = excluded.inclusion_watermark, + total_connections = excluded.total_connections, + in_flight_connections = excluded.in_flight_connections, + reserved_connection_units = excluded.reserved_connection_units, + enforced_connection_units = excluded.enforced_connection_units, + snapshot_at = excluded.snapshot_at`, + [ + input.cellId, + input.cellIncarnation, + inclusionWatermark, + input.totalConnections, + input.inFlightConnections, + input.reservedConnectionUnits, + input.enforcedConnectionUnits, + now + ] + ) + } + await transaction.query(`DELETE FROM relay_cell_fences WHERE cell_id = ?`, [input.cellId]) + await transaction.query(`DELETE FROM relay_cell_committed_fences WHERE cell_id = ?`, [ + input.cellId + ]) + await transaction.query( + `DELETE FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + [input.cellId] + ) + }) + if (inclusionWatermark !== undefined) { + await this.releaseHeartbeatConnectionReservationDebt( + input.cellId, + input.cellIncarnation, + inclusionWatermark, + now + ) + } + } + + async recordCellRegionalRehomeStatus(input: CellRegionalRehomeStatus): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + const runtime = ( + await transaction.queryLocked( + `SELECT cell_incarnation FROM relay_cell_runtime WHERE cell_id = ?`, + [input.cellId] + ) + )[0] + if (!runtime || text(runtime, 'cell_incarnation') !== input.cellIncarnation) { + throw new Error('stale_cell_incarnation') + } + await transaction.query( + `INSERT INTO relay_cell_capabilities + (cell_id, cell_incarnation, regional_rehome_protocol, last_heartbeat_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + regional_rehome_protocol = excluded.regional_rehome_protocol, + last_heartbeat_at = excluded.last_heartbeat_at`, + [input.cellId, input.cellIncarnation, input.regionalRehomeProtocol, now] + ) + const safety = input.safety + await transaction.query( + `INSERT INTO relay_cell_rehome_safety + (cell_id, cell_incarnation, observed_at, sql_failures, reconnects, + control_activity_recovery_failures, database_pool_waiting, + database_pool_waiters_max, database_pool_wait_ms_max) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + observed_at = excluded.observed_at, + sql_failures = excluded.sql_failures, + reconnects = excluded.reconnects, + control_activity_recovery_failures = + excluded.control_activity_recovery_failures, + database_pool_waiting = excluded.database_pool_waiting, + database_pool_waiters_max = excluded.database_pool_waiters_max, + database_pool_wait_ms_max = excluded.database_pool_wait_ms_max`, + [ + input.cellId, + input.cellIncarnation, + safety.observedAt, + safety.sqlFailures, + safety.reconnects, + safety.controlActivityRecoveryFailures, + safety.databasePoolWaiting, + safety.databasePoolWaitersMax, + safety.databasePoolWaitMsMax + ] + ) + }) + } + + private async releaseHeartbeatConnectionReservationDebt( + cellId: string, + cellIncarnation: string, + inclusionWatermark: number, + now: number + ): Promise { + await this.database.transaction(async (transaction) => { + const runtime = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, + [cellId] + ) + )[0] + const snapshot = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_connection_snapshots WHERE cell_id = ?`, + [cellId] + ) + )[0] + if ( + !runtime || + !snapshot || + text(runtime, 'cell_incarnation') !== cellIncarnation || + text(snapshot, 'cell_incarnation') !== cellIncarnation || + integer(snapshot, 'inclusion_watermark') !== inclusionWatermark + ) { + return + } + await transaction.query( + `UPDATE relay_control_connection_reservations + SET state = 'released', released_at = ?, updated_at = ? + WHERE cell_id = ? AND state = 'claimed' + AND inclusion_watermark IS NOT NULL + AND inclusion_watermark <= ?`, + [now, now, cellId, inclusionWatermark] + ) + await transaction.query( + `UPDATE relay_control_connection_reservations + SET state = 'released', released_at = ?, updated_at = ? + WHERE reservation_id IN ( + SELECT duplicate.reservation_id + FROM relay_control_connection_reservations duplicate + WHERE duplicate.cell_id = ? + AND duplicate.state = 'late-arrival-debt' + AND duplicate.claim_activity_id IS NULL + AND duplicate.timeout_at <= ? + AND EXISTS ( + SELECT 1 + FROM relay_control_connection_reservations retained + WHERE retained.user_id = duplicate.user_id + AND retained.relay_host_id = duplicate.relay_host_id + AND retained.assignment_epoch = duplicate.assignment_epoch + AND retained.cell_id = duplicate.cell_id + AND retained.state = 'late-arrival-debt' + AND retained.claim_activity_id IS NULL + AND retained.timeout_at <= ? + AND ( + retained.created_at < duplicate.created_at OR + ( + retained.created_at = duplicate.created_at AND + retained.reservation_id < duplicate.reservation_id + ) + ) + ) + )`, + [now, now, cellId, now, now] + ) + await transaction.query( + `UPDATE relay_control_connection_reservations + SET state = 'released', released_at = ?, updated_at = ? + WHERE cell_id = ? AND state = 'late-arrival-debt' + AND claim_activity_id IS NULL AND timeout_at <= ? + AND EXISTS ( + SELECT 1 FROM relay_assignments assignment + WHERE assignment.user_id = + relay_control_connection_reservations.user_id + AND assignment.relay_host_id = + relay_control_connection_reservations.relay_host_id + AND assignment.assignment_epoch > + relay_control_connection_reservations.assignment_epoch + ) + AND EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = + relay_control_connection_reservations.user_id + AND migration.relay_host_id = + relay_control_connection_reservations.relay_host_id + AND migration.assignment_epoch = + relay_control_connection_reservations.assignment_epoch + AND migration.target_cell_id = + relay_control_connection_reservations.cell_id + AND migration.aborted_at IS NOT NULL + )`, + [now, now, cellId, now] + ) + }) + } + + async attestCellFence(cellId: string, cellIncarnation: string): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const cell = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId]) + )[0] + if (!cell) throw new Error('cell_not_found') + if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled') + const runtime = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + cellId + ]) + )[0] + if ( + !runtime || + text(runtime, 'cell_incarnation') !== cellIncarnation || + integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs + ) { + throw new Error('cell_fence_runtime_not_stale') + } + const expiresAt = now + CELL_FENCE_TTL_MS + await transaction.query( + `INSERT INTO relay_cell_fences + (cell_id, cell_incarnation, attested_at, expires_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + attested_at = excluded.attested_at, + expires_at = excluded.expires_at`, + [cellId, cellIncarnation, now, expiresAt] + ) + return expiresAt + }) + } + + async adoptLegacyCellFence(cellId: string, cellIncarnation: string): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const cell = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId]) + )[0] + if (!cell) throw new Error('cell_not_found') + if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled') + const runtime = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + cellId + ]) + )[0] + if ( + !runtime || + text(runtime, 'cell_incarnation') !== cellIncarnation || + integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs + ) { + throw new Error('cell_fence_runtime_not_stale') + } + const attempt = ( + await transaction.queryLocked( + `SELECT attempt_id FROM relay_cell_fence_attempts + WHERE cell_id = ? ORDER BY created_at DESC LIMIT 1`, + [cellId] + ) + )[0] + if (attempt) throw new Error('legacy_cell_fence_attempt_exists') + const expiresAt = now + CELL_FENCE_TTL_MS + await transaction.query( + `INSERT INTO relay_cell_fences + (cell_id, cell_incarnation, attested_at, expires_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + attested_at = excluded.attested_at, + expires_at = excluded.expires_at`, + [cellId, cellIncarnation, now, expiresAt] + ) + return expiresAt + }) + } + + async commitLegacyCellFenceAdoption( + cellId: string, + cellIncarnation: string + ): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + const cell = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId]) + )[0] + if (!cell) throw new Error('cell_not_found') + if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled') + const runtime = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + cellId + ]) + )[0] + const fence = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_fences WHERE cell_id = ?`, [ + cellId + ]) + )[0] + const attempt = ( + await transaction.queryLocked( + `SELECT attempt_id FROM relay_cell_fence_attempts + WHERE cell_id = ? ORDER BY created_at DESC LIMIT 1`, + [cellId] + ) + )[0] + if (attempt) throw new Error('legacy_cell_fence_attempt_exists') + if ( + !runtime || + !fence || + text(runtime, 'cell_incarnation') !== cellIncarnation || + text(fence, 'cell_incarnation') !== cellIncarnation || + integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs || + integer(fence, 'attested_at') < integer(runtime, 'last_heartbeat_at') || + integer(fence, 'expires_at') <= now + ) { + throw new Error('legacy_cell_fence_not_active') + } + await transaction.query( + `INSERT INTO relay_cell_legacy_fence_adoptions + (cell_id, cell_incarnation, attested_at, expires_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + attested_at = excluded.attested_at, + expires_at = excluded.expires_at`, + [cellId, cellIncarnation, now, integer(fence, 'expires_at')] + ) + }) + } + + async prepareCellFenceAttempt(input: CellFenceAttemptEvidence): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + let createdAt = now + const cell = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [ + input.cellId + ]) + )[0] + if (!cell) throw new Error('cell_not_found') + if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled') + const runtime = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + input.cellId + ]) + )[0] + if (!runtime || text(runtime, 'cell_incarnation') !== input.cellIncarnation) { + throw new Error('cell_fence_runtime_mismatch') + } + const existing = ( + await transaction.queryLocked( + `${CELL_FENCE_ATTEMPT_SELECT} + WHERE attempts.cell_id = ? ORDER BY attempts.created_at DESC LIMIT 1`, + [input.cellId] + ) + )[0] + if (existing) { + const attempt = cellFenceAttempt(existing) + if (!attempt.completedAt && !attempt.abortedAt) { + assertCellFenceAttemptBase(existing, input) + return attempt + } + createdAt = Math.max(now, attempt.createdAt + 1) + } else { + const activeFence = ( + await transaction.queryLocked( + `SELECT cell_id FROM relay_cell_fences + WHERE cell_id = ? AND cell_incarnation = ? AND expires_at > ?`, + [input.cellId, input.cellIncarnation, now] + ) + )[0] + if (activeFence) throw new Error('cell_fence_already_attested') + } + const expiresAt = createdAt + CELL_FENCE_ATTEMPT_TTL_MS + await transaction.query( + `INSERT INTO relay_cell_fence_attempts + (attempt_id, environment, cell_id, cell_incarnation, mig_name, instance_group, + generation_identity, fence_commit, plan_sha256, gce_operation, created_at, + expires_at, apply_started_at, completed_at, aborted_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, NULL, NULL, NULL)`, + [ + input.attemptId, + input.environment, + input.cellId, + input.cellIncarnation, + input.migName, + input.instanceGroup, + input.generationIdentity, + input.fenceCommit, + input.planSha256, + createdAt, + expiresAt + ] + ) + await transaction.query( + `INSERT INTO relay_cell_fence_plan_bindings + (attempt_id, plan_object_name, plan_object_generation, var_file_sha256, + terraform_state_lineage, terraform_state_serial, + terraform_state_object_generation, terraform_state_object_sha256, + request_reason) + VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?)`, + [ + input.attemptId, + input.planObjectName, + input.varFileSha256, + input.terraformStateLineage, + input.terraformStateSerial, + input.terraformStateObjectGeneration, + input.terraformStateObjectSha256, + input.requestReason + ] + ) + const { planObjectGeneration: _ignored, ...prepared } = input + return { ...prepared, createdAt, expiresAt } + }) + } + + async bindCellFencePlanGeneration( + input: CellFenceAttemptEvidence, + planObjectGeneration: string + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const row = await lockedCellFenceAttempt(transaction, input.attemptId) + assertCellFenceAttemptBase(row, input) + if (integer(row, 'expires_at') <= now) throw new Error('cell_fence_attempt_expired') + if (row.apply_started_at !== null) throw new Error('cell_fence_apply_already_started') + if (row.completed_at !== null || row.aborted_at !== null) { + throw new Error('cell_fence_attempt_terminal') + } + const existing = optionalText(row, 'plan_object_generation') + if (existing && existing !== planObjectGeneration) { + throw new Error('cell_fence_plan_generation_mismatch') + } + if (!existing) { + await transaction.query( + `UPDATE relay_cell_fence_plan_bindings + SET plan_object_generation = ? WHERE attempt_id = ?`, + [planObjectGeneration, input.attemptId] + ) + row.plan_object_generation = planObjectGeneration + } + return cellFenceAttempt(row) + }) + } + + async startCellFenceApply( + input: CellFenceAttemptEvidence, + invocationId: string, + requestReason: string + ): Promise<{ attempt: CellFenceAttempt; invocation: CellFenceApplyInvocation }> { + const now = this.now() + return await this.database.transaction(async (transaction) => { + if (requestReason !== `${input.requestReason}/${invocationId}`) { + throw new Error('cell_fence_invocation_mismatch') + } + const row = await lockedCellFenceAttempt(transaction, input.attemptId) + assertActiveCellFenceAttempt(row, input, now) + const existing = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_fence_apply_invocations WHERE invocation_id = ?`, + [invocationId] + ) + )[0] + if (existing) { + if ( + text(existing, 'attempt_id') !== input.attemptId || + text(existing, 'request_reason') !== requestReason + ) { + throw new Error('cell_fence_invocation_mismatch') + } + return { + attempt: cellFenceAttempt(row), + invocation: cellFenceApplyInvocation(existing) + } + } + if (row.apply_started_at === null) { + await transaction.query( + `UPDATE relay_cell_fence_attempts SET apply_started_at = ? WHERE attempt_id = ?`, + [now, input.attemptId] + ) + row.apply_started_at = now + } + await transaction.query( + `INSERT INTO relay_cell_fence_apply_invocations + (invocation_id, attempt_id, request_reason, started_at, gce_operation) + VALUES (?, ?, ?, ?, NULL)`, + [invocationId, input.attemptId, requestReason, now] + ) + return { + attempt: cellFenceAttempt(row), + invocation: { invocationId, requestReason, startedAt: now } + } + }) + } + + async recordCellFenceOperation( + input: CellFenceAttemptEvidence, + invocationId: string, + requestReason: string, + gceOperation: string + ): Promise<{ attempt: CellFenceAttempt; invocation: CellFenceApplyInvocation }> { + const now = this.now() + return await this.database.transaction(async (transaction) => { + if (requestReason !== `${input.requestReason}/${invocationId}`) { + throw new Error('cell_fence_invocation_mismatch') + } + const row = await lockedCellFenceAttempt(transaction, input.attemptId) + assertActiveCellFenceAttempt(row, input, now) + if (row.apply_started_at === null) throw new Error('cell_fence_apply_not_started') + const invocation = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_fence_apply_invocations WHERE invocation_id = ?`, + [invocationId] + ) + )[0] + if ( + !invocation || + text(invocation, 'attempt_id') !== input.attemptId || + text(invocation, 'request_reason') !== requestReason + ) { + throw new Error('cell_fence_invocation_mismatch') + } + const existing = invocation.gce_operation + if (existing !== null && existing !== gceOperation) { + throw new Error('cell_fence_operation_mismatch') + } + await transaction.query( + `UPDATE relay_cell_fence_apply_invocations + SET gce_operation = ? WHERE invocation_id = ?`, + [gceOperation, invocationId] + ) + invocation.gce_operation = gceOperation + await transaction.query( + `UPDATE relay_cell_fence_attempts SET gce_operation = ? WHERE attempt_id = ?`, + [gceOperation, input.attemptId] + ) + row.gce_operation = gceOperation + return { + attempt: cellFenceAttempt(row), + invocation: cellFenceApplyInvocation(invocation) + } + }) + } + + async cellFenceAttempt(cellId: string): Promise { + const rows = await this.database.query( + `${CELL_FENCE_ATTEMPT_SELECT} + WHERE attempts.cell_id = ? ORDER BY attempts.created_at DESC LIMIT 1`, + [cellId] + ) + if (rows.length === 0) return null + const attempt = cellFenceAttempt(rows[0]!) + const invocations = await this.database.query( + `SELECT * FROM relay_cell_fence_apply_invocations + WHERE attempt_id = ? ORDER BY started_at, invocation_id`, + [attempt.attemptId] + ) + return { + ...attempt, + applyInvocations: invocations.map(cellFenceApplyInvocation) + } + } + + async abortCellFenceAttempt(input: CellFenceAttemptEvidence): Promise { + return await this.database.transaction(async (transaction) => { + const row = await lockedCellFenceAttempt(transaction, input.attemptId) + assertCellFenceAttemptBase(row, input) + if (row.completed_at !== null) throw new Error('cell_fence_attempt_completed') + if (row.aborted_at !== null) throw new Error('cell_fence_attempt_aborted') + if (row.apply_started_at !== null || row.gce_operation !== null) { + throw new Error('cell_fence_apply_may_have_started') + } + const abortedAt = this.now() + await transaction.query( + `UPDATE relay_cell_fence_attempts SET aborted_at = ? WHERE attempt_id = ?`, + [abortedAt, input.attemptId] + ) + row.aborted_at = abortedAt + return cellFenceAttempt(row) + }) + } + + async attestCellFenceAttempt( + input: CellFenceAttemptEvidence, + gceOperation: string + ): Promise<{ expiresAt: number; attempt: CellFenceAttempt }> { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const attemptRow = await lockedCellFenceAttempt(transaction, input.attemptId) + assertCellFenceAttemptEvidence(attemptRow, input) + if (attemptRow.completed_at !== null) { + if (attemptRow.gce_operation !== gceOperation) { + throw new Error('cell_fence_operation_not_attested') + } + const cell = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [ + input.cellId + ]) + )[0] + if (!cell) throw new Error('cell_not_found') + if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled') + const runtime = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + input.cellId + ]) + )[0] + if ( + !runtime || + text(runtime, 'cell_incarnation') !== input.cellIncarnation || + integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs + ) { + throw new Error('cell_fence_runtime_not_stale') + } + const expiresAt = now + CELL_FENCE_TTL_MS + await transaction.query( + `INSERT INTO relay_cell_fences + (cell_id, cell_incarnation, attested_at, expires_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + attested_at = excluded.attested_at, + expires_at = excluded.expires_at`, + [input.cellId, input.cellIncarnation, now, expiresAt] + ) + await this.recordCommittedCellFence( + transaction, + input.cellId, + input.cellIncarnation, + input.attemptId, + now, + expiresAt + ) + return { + expiresAt, + attempt: cellFenceAttempt(attemptRow) + } + } + assertActiveCellFenceAttempt(attemptRow, input, now) + if ( + attemptRow.apply_started_at === null || + attemptRow.gce_operation !== gceOperation + ) { + throw new Error('cell_fence_operation_not_attested') + } + const cell = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [ + input.cellId + ]) + )[0] + if (!cell) throw new Error('cell_not_found') + if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled') + const runtime = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + input.cellId + ]) + )[0] + if ( + !runtime || + text(runtime, 'cell_incarnation') !== input.cellIncarnation || + integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs + ) { + throw new Error('cell_fence_runtime_not_stale') + } + const expiresAt = now + CELL_FENCE_TTL_MS + await transaction.query( + `INSERT INTO relay_cell_fences + (cell_id, cell_incarnation, attested_at, expires_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, + attested_at = excluded.attested_at, + expires_at = excluded.expires_at`, + [input.cellId, input.cellIncarnation, now, expiresAt] + ) + await this.recordCommittedCellFence( + transaction, + input.cellId, + input.cellIncarnation, + input.attemptId, + now, + expiresAt + ) + await transaction.query( + `UPDATE relay_cell_fence_attempts SET completed_at = ? WHERE attempt_id = ?`, + [now, input.attemptId] + ) + attemptRow.completed_at = now + return { expiresAt, attempt: cellFenceAttempt(attemptRow) } + }) + } + + async prepareCellDrainAttempt(input: { + attemptId: string + cellId: string + cellIncarnation: string + traceValue: string + plannedGraceMs: number + }): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + await this.assertDrainCellGeneration( + transaction, + input.cellId, + input.cellIncarnation + ) + const existing = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_drain_attempt_states + WHERE cell_id = ? ORDER BY prepared_at DESC LIMIT 1`, + [input.cellId] + ) + )[0] + if (existing) { + if (text(existing, 'attempt_id') === input.attemptId) { + if ( + text(existing, 'cell_incarnation') !== input.cellIncarnation || + text(existing, 'trace_value') !== input.traceValue || + integer(existing, 'planned_grace_ms') !== input.plannedGraceMs + ) { + throw new Error('drain_attempt_generation_mismatch') + } + return { ...cellDrainAttempt(existing), shouldSend: false } + } + if ( + text(existing, 'state') !== 'proven-not-delivered' || + text(existing, 'cell_incarnation') !== input.cellIncarnation + ) { + throw new Error('drain_attempt_generation_mismatch') + } + } + await transaction.query( + `INSERT INTO relay_cell_drain_attempt_states + (attempt_id, cell_id, cell_incarnation, trace_value, planned_grace_ms, + state, prepared_at, send_may_have_started_at, send_permit_expires_at, + application_receipt_at, backend_success_status, backend_instance, + receipt_cell_incarnation, retry_after, recover_forward_attempted_at, + proven_not_delivered_at) + VALUES (?, ?, ?, ?, ?, 'prepared', ?, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL)`, + [ + input.attemptId, + input.cellId, + input.cellIncarnation, + input.traceValue, + input.plannedGraceMs, + now + ] + ) + return { + attemptId: input.attemptId, + cellId: input.cellId, + cellIncarnation: input.cellIncarnation, + traceValue: input.traceValue, + plannedGraceMs: input.plannedGraceMs, + state: 'prepared', + preparedAt: now, + shouldSend: false + } + }) + } + + async beginCellDrainSend(input: { + attemptId: string + cellId: string + cellIncarnation: string + }): Promise { + try { + return await this.beginCellDrainSendOnce(input) + } catch (error) { + if ( + !(error instanceof Error) || + error.message !== 'migration_activity_accounting_mismatch' + ) { + throw error + } + const activityCells = await this.database.query( + `SELECT DISTINCT lease.cell_id + FROM relay_assignment_activity_leases lease + WHERE EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = lease.user_id + AND migration.relay_host_id = lease.relay_host_id + AND migration.source_cell_id = ? + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ORDER BY lease.cell_id`, + [input.cellId] + ) + const cellIds = activityCells + .map((row) => text(row, 'cell_id')) + .filter((cellId) => cellId !== input.cellId) + for (const cellId of cellIds) { + await this.reconcileReservationAccounting(input.cellId, cellId) + } + await this.refreshDrainMigrationLeases(input) + return await this.beginCellDrainSendOnce(input) + } + } + + private async refreshDrainMigrationLeases(input: { + cellId: string + cellIncarnation: string + }): Promise { + await this.retireObsoleteDrainMigrations(input.cellId) + for (let repairAttempt = 0; ; repairAttempt++) { + try { + await this.refreshDrainMigrationLeasesOnce(input) + return + } catch (error) { + if ( + !(error instanceof Error) || + error.message !== 'migration_activity_accounting_mismatch' || + repairAttempt >= MAX_DRAIN_ACCOUNTING_REPAIR_ATTEMPTS + ) { + throw error + } + await this.reconcileDrainMigrationAccounting(input.cellId) + } + } + } + + private async reconcileDrainMigrationAccounting(sourceCellId: string): Promise { + const activityCells = await this.database.query( + `SELECT DISTINCT lease.cell_id + FROM relay_assignment_activity_leases lease + WHERE EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = lease.user_id + AND migration.relay_host_id = lease.relay_host_id + AND migration.source_cell_id = ? + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ORDER BY lease.cell_id`, + [sourceCellId] + ) + for (const cellId of activityCells + .map((row) => text(row, 'cell_id')) + .filter((cellId) => cellId !== sourceCellId)) { + await this.reconcileReservationAccounting(sourceCellId, cellId) + } + } + + private async retireObsoleteDrainMigrations(sourceCellId: string): Promise { + const candidates = await this.database.query( + `SELECT migration.user_id, migration.relay_host_id, + migration.assignment_epoch + FROM relay_assignment_migrations migration + JOIN relay_assignments assignment + ON assignment.user_id = migration.user_id + AND assignment.relay_host_id = migration.relay_host_id + WHERE migration.source_cell_id = ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + AND assignment.assignment_epoch > migration.assignment_epoch + ORDER BY migration.user_id, migration.relay_host_id`, + [sourceCellId] + ) + for (const candidate of candidates) { + await this.retireObsoleteDrainMigration(sourceCellId, candidate) + } + } + + private async retireObsoleteDrainMigration( + sourceCellId: string, + candidate: SqlRow + ): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + const identity = { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + } + const assignment = await this.assignmentRow(transaction, identity) + const assignmentEpoch = integer(candidate, 'assignment_epoch') + const migrationRow = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND source_cell_id = ? + AND completed_at IS NULL AND aborted_at IS NULL`, + [identity.userId, identity.relayHostId, assignmentEpoch, sourceCellId] + ) + )[0] + if (!migrationRow) return + if (!assignment || integer(assignment, 'assignment_epoch') <= assignmentEpoch) { + throw new Error('migration_assignment_mismatch') + } + const leases = await this.lockAssignmentActivities(transaction, identity) + const obsoleteActivityIds = new Set([ + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ]) + if (leases.some((lease) => obsoleteActivityIds.has(text(lease, 'activity_id')))) { + throw new Error('migration_activity_topology_mismatch') + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + text(migrationRow, 'target_cell_id'), + assignmentEpoch, + now + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + }) + } + + private async refreshDrainMigrationLeasesOnce(input: { + cellId: string + cellIncarnation: string + }): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + await this.assertDrainCellGeneration( + transaction, + input.cellId, + input.cellIncarnation + ) + const assignments = await transaction.queryLocked( + `SELECT assignment.* + FROM relay_assignments assignment + WHERE EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + AND migration.source_cell_id = ? + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ORDER BY assignment.user_id, assignment.relay_host_id`, + [input.cellId] + ) + const activityLeases = await transaction.queryLocked( + `SELECT lease.* + FROM relay_assignment_activity_leases lease + WHERE EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = lease.user_id + AND migration.relay_host_id = lease.relay_host_id + AND migration.source_cell_id = ? + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id`, + [input.cellId] + ) + const migrations = await transaction.queryLocked( + `SELECT migration.* + FROM relay_assignment_migrations migration + WHERE migration.source_cell_id = ? + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ORDER BY migration.user_id, migration.relay_host_id`, + [input.cellId] + ) + const cells = await this.lockCellInventory(transaction) + for (const migrationRow of migrations) { + const identity = { + userId: text(migrationRow, 'user_id'), + relayHostId: text(migrationRow, 'relay_host_id') + } + const assignment = assignments.find( + (candidate) => + text(candidate, 'user_id') === identity.userId && + text(candidate, 'relay_host_id') === identity.relayHostId + ) + assertCurrentMigrationAssignment(assignment, migrationRow) + const leases = activityLeases.filter( + (lease) => + text(lease, 'user_id') === identity.userId && + text(lease, 'relay_host_id') === identity.relayHostId + ) + const migrationLeases = leases.filter( + (lease) => text(lease, 'activity_kind') === 'migration' + ) + const assignmentEpoch = integer(migrationRow, 'assignment_epoch') + const sourceRequestUnits = integer(migrationRow, 'source_request_units') + const targetCellId = text(migrationRow, 'target_cell_id') + const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs + if (migrationLeases.length > 0) { + assertAssignmentActivityAccounting(assignment, leases, migrationRow) + await transaction.query( + `UPDATE relay_assignment_activity_leases SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? + AND activity_id IN (?, ?)`, + [ + expiresAt, + now, + identity.userId, + identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ] + ) + await transaction.query( + `UPDATE relay_assignments SET + lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, + last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [expiresAt, expiresAt, now, identity.userId, identity.relayHostId] + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + await this.refreshPendingControlReservation( + transaction, + identity, + targetCellId, + assignmentEpoch, + expiresAt, + now + ) + continue + } + if ( + optionalInteger(migrationRow, 'target_registered_at') === undefined || + integer(migrationRow, 'expires_at') > now || + sourceRequestUnits < 0 || + integer(migrationRow, 'target_reserved_units') !== sourceRequestUnits + 1 || + activityLeaseById(leases, migrationActivityId(assignmentEpoch)) || + !cells.some((cell) => text(cell, 'cell_id') === targetCellId) + ) { + throw new Error('migration_activity_lease_shape_mismatch') + } + await this.adjustCellReservation(transaction, targetCellId, sourceRequestUnits) + await transaction.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, 'migration', ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + migrationActivityId(assignmentEpoch), + targetCellId, + sourceRequestUnits, + expiresAt, + now + ] + ) + await transaction.query( + `UPDATE relay_assignments SET migration_leases = migration_leases + 1, + lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, + last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [expiresAt, expiresAt, now, identity.userId, identity.relayHostId] + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + await this.refreshPendingControlReservation( + transaction, + identity, + targetCellId, + assignmentEpoch, + expiresAt, + now + ) + } + }) + } + + private async beginCellDrainSendOnce(input: { + attemptId: string + cellId: string + cellIncarnation: string + }): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + await this.assertDrainCellGeneration( + transaction, + input.cellId, + input.cellIncarnation + ) + const row = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_drain_attempt_states + WHERE attempt_id = ? AND cell_id = ?`, + [input.attemptId, input.cellId] + ) + )[0] + if (!row || text(row, 'cell_incarnation') !== input.cellIncarnation) { + throw new Error('drain_attempt_not_found') + } + if (text(row, 'state') !== 'prepared') { + return { ...cellDrainAttempt(row), shouldSend: false } + } + const assignments = await transaction.queryLocked( + `SELECT assignment.* + FROM relay_assignments assignment + JOIN relay_assignment_migrations migration + ON migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + WHERE migration.source_cell_id = ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + ORDER BY assignment.user_id, assignment.relay_host_id`, + [input.cellId] + ) + const activityLeases = await transaction.queryLocked( + `SELECT lease.* + FROM relay_assignment_activity_leases lease + WHERE EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = lease.user_id + AND migration.relay_host_id = lease.relay_host_id + AND migration.source_cell_id = ? + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id`, + [input.cellId] + ) + const migrationIncarnations = await transaction.queryLocked( + `SELECT migration.*, + ( + SELECT incarnation.source_cell_incarnation + FROM relay_assignment_migration_incarnations incarnation + WHERE incarnation.user_id = migration.user_id + AND incarnation.relay_host_id = migration.relay_host_id + AND incarnation.assignment_epoch = migration.assignment_epoch + ) AS source_cell_incarnation, + ( + SELECT incarnation.target_cell_incarnation + FROM relay_assignment_migration_incarnations incarnation + WHERE incarnation.user_id = migration.user_id + AND incarnation.relay_host_id = migration.relay_host_id + AND incarnation.assignment_epoch = migration.assignment_epoch + ) AS target_cell_incarnation + FROM relay_assignment_migrations migration + WHERE migration.source_cell_id = ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + ORDER BY migration.user_id, migration.relay_host_id`, + [input.cellId] + ) + if ( + migrationIncarnations.some( + (migration) => + optionalText(migration, 'source_cell_incarnation') !== input.cellIncarnation + ) + ) { + throw new Error('drain_migration_source_incarnation_mismatch') + } + for (const migrationRow of migrationIncarnations) { + const assignment = assignments.find( + (candidate) => + text(candidate, 'user_id') === text(migrationRow, 'user_id') && + text(candidate, 'relay_host_id') === text(migrationRow, 'relay_host_id') + ) + assertCurrentMigrationAssignment(assignment, migrationRow) + assertAssignmentActivityAccounting( + assignment, + activityLeases.filter( + (lease) => + text(lease, 'user_id') === text(migrationRow, 'user_id') && + text(lease, 'relay_host_id') === text(migrationRow, 'relay_host_id') + ), + migrationRow + ) + } + const sendPermitExpiresAt = now + CELL_DRAIN_SEND_PERMIT_MS + await transaction.query( + `UPDATE relay_cell_drain_attempt_states + SET state = 'send-may-have-started', send_may_have_started_at = ?, + send_permit_expires_at = ? + WHERE attempt_id = ?`, + [now, sendPermitExpiresAt, input.attemptId] + ) + await transaction.query( + `INSERT INTO relay_post_drain_migration_pins + (user_id, relay_host_id, assignment_epoch, drain_attempt_id, + source_cell_id, source_cell_incarnation, target_cell_id, + target_cell_incarnation, source_request_units, target_reserved_units, + pinned_at) + SELECT migration.user_id, migration.relay_host_id, + migration.assignment_epoch, ?, migration.source_cell_id, + incarnation.source_cell_incarnation, migration.target_cell_id, + incarnation.target_cell_incarnation, migration.source_request_units, + migration.target_reserved_units, ? + FROM relay_assignment_migrations migration + JOIN relay_assignment_migration_incarnations incarnation + ON incarnation.user_id = migration.user_id + AND incarnation.relay_host_id = migration.relay_host_id + AND incarnation.assignment_epoch = migration.assignment_epoch + WHERE migration.source_cell_id = ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + ON CONFLICT (user_id, relay_host_id, assignment_epoch) DO NOTHING`, + [input.attemptId, now, input.cellId] + ) + row.state = 'send-may-have-started' + row.send_may_have_started_at = now + row.send_permit_expires_at = sendPermitExpiresAt + return { ...cellDrainAttempt(row), shouldSend: true } + }) + } + + async recordCellDrainApplicationReceipt(input: { + attemptId: string + cellId: string + cellIncarnation: string + traceValue: string + backendStatus: number + backendInstance?: string + }): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + await this.assertDrainCellGeneration( + transaction, + input.cellId, + input.cellIncarnation + ) + const row = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_drain_attempt_states + WHERE attempt_id = ? AND cell_id = ?`, + [input.attemptId, input.cellId] + ) + )[0] + if ( + !row || + text(row, 'cell_incarnation') !== input.cellIncarnation || + text(row, 'trace_value') !== input.traceValue + ) { + throw new Error('drain_attempt_not_found') + } + if (text(row, 'state') === 'application-receipt') return cellDrainAttempt(row) + if ( + text(row, 'state') !== 'send-may-have-started' || + input.backendStatus < 200 || + input.backendStatus >= 300 + ) { + throw new Error('drain_application_receipt_invalid') + } + const retryAfter = now + integer(row, 'planned_grace_ms') + 30_000 + await transaction.query( + `UPDATE relay_cell_drain_attempt_states + SET state = 'application-receipt', application_receipt_at = ?, + backend_success_status = ?, backend_instance = ?, + receipt_cell_incarnation = ?, retry_after = ? + WHERE attempt_id = ?`, + [ + now, + input.backendStatus, + input.backendInstance, + input.cellIncarnation, + retryAfter, + input.attemptId + ] + ) + row.state = 'application-receipt' + row.application_receipt_at = now + row.backend_success_status = input.backendStatus + row.backend_instance = input.backendInstance ?? null + row.receipt_cell_incarnation = input.cellIncarnation + row.retry_after = retryAfter + return cellDrainAttempt(row) + }) + } + + async proveCellDrainNotDelivered(input: { + attemptId: string + cellId: string + cellIncarnation: string + }): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const row = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_drain_attempt_states + WHERE attempt_id = ? AND cell_id = ?`, + [input.attemptId, input.cellId] + ) + )[0] + if (!row || text(row, 'cell_incarnation') !== input.cellIncarnation) { + throw new Error('drain_attempt_not_found') + } + if (text(row, 'state') === 'proven-not-delivered') return cellDrainAttempt(row) + if (text(row, 'state') !== 'send-may-have-started') { + throw new Error('drain_delivery_proof_invalid') + } + await transaction.query( + `UPDATE relay_cell_drain_attempt_states + SET state = 'proven-not-delivered', proven_not_delivered_at = ? + WHERE attempt_id = ?`, + [now, input.attemptId] + ) + row.state = 'proven-not-delivered' + row.proven_not_delivered_at = now + return cellDrainAttempt(row) + }) + } + + async prepareCellDrainRecovery(input: { + attemptId?: string + cellId: string + cellIncarnation: string + }): Promise<{ + shouldSend: boolean + retryAfter: number + preparedAttempt?: CellDrainAttempt + }> { + await this.refreshDrainMigrationLeases(input) + const now = this.now() + return await this.database.transaction(async (transaction) => { + await this.assertDrainCellGeneration( + transaction, + input.cellId, + input.cellIncarnation + ) + const attempt = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_drain_attempt_states + WHERE cell_id = ? + ORDER BY prepared_at DESC LIMIT 1`, + [input.cellId] + ) + )[0] + if (!attempt || (input.attemptId && text(attempt, 'attempt_id') !== input.attemptId)) { + throw new Error('drain_application_receipt_missing') + } + if (text(attempt, 'state') === 'prepared') { + if (text(attempt, 'cell_incarnation') !== input.cellIncarnation) { + throw new Error('drain_application_receipt_missing') + } + return { + shouldSend: false, + retryAfter: now, + preparedAttempt: cellDrainAttempt(attempt) + } + } + const applicationReceiptAt = optionalInteger(attempt, 'application_receipt_at') + const backendSuccessStatus = optionalInteger(attempt, 'backend_success_status') + const retryAfter = optionalInteger(attempt, 'retry_after') + if ( + text(attempt, 'state') !== 'application-receipt' || + optionalText(attempt, 'receipt_cell_incarnation') !== + text(attempt, 'cell_incarnation') || + applicationReceiptAt === undefined || + backendSuccessStatus === undefined || + backendSuccessStatus < 200 || + backendSuccessStatus >= 300 || + retryAfter === undefined + ) { + throw new Error('drain_application_receipt_missing') + } + if (now < retryAfter) throw new Error('drain_recovery_too_early') + const attemptId = text(attempt, 'attempt_id') + if (text(attempt, 'cell_incarnation') !== input.cellIncarnation) { + const priorRecovery = ( + await transaction.queryLocked( + `SELECT attempted_at FROM relay_cell_drain_recovery_attempts + WHERE drain_attempt_id = ? AND cell_incarnation = ?`, + [attemptId, input.cellIncarnation] + ) + )[0] + if (priorRecovery) return { shouldSend: false, retryAfter } + await transaction.query( + `INSERT INTO relay_cell_drain_recovery_attempts + (drain_attempt_id, cell_incarnation, attempted_at) VALUES (?, ?, ?)`, + [attemptId, input.cellIncarnation, now] + ) + return { shouldSend: true, retryAfter } + } + if (optionalInteger(attempt, 'recover_forward_attempted_at') !== undefined) { + return { shouldSend: false, retryAfter } + } + await transaction.query( + `UPDATE relay_cell_drain_attempt_states SET recover_forward_attempted_at = ? + WHERE attempt_id = ? AND recover_forward_attempted_at IS NULL`, + [now, attemptId] + ) + return { shouldSend: true, retryAfter } + }) + } + + async evacuateDeadCells(limit = 100): Promise { + if (!this.requireLiveCells) return 0 + const cutoff = this.now() - this.heartbeatTtlMs + const rows = await this.database.query( + `SELECT assignment.user_id, assignment.relay_host_id, assignment.cell_id + FROM relay_assignments assignment + JOIN relay_cells cell ON cell.cell_id = assignment.cell_id + LEFT JOIN relay_cell_committed_fences committed + ON committed.cell_id = assignment.cell_id + LEFT JOIN relay_cell_fence_attempts attempt + ON attempt.attempt_id = committed.attempt_id + LEFT JOIN relay_cell_fences fence ON fence.cell_id = assignment.cell_id + LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id + WHERE (runtime.cell_id IS NULL OR runtime.ready != ? OR runtime.last_heartbeat_at <= ?) + AND ( + ( + cell.enabled = 1 + AND NOT EXISTS ( + SELECT 1 FROM relay_cell_connection_limits limits + WHERE limits.cell_id = assignment.cell_id + ) + AND NOT EXISTS ( + SELECT 1 FROM relay_post_drain_migration_pins pin + JOIN relay_assignment_migrations migration + ON migration.user_id = pin.user_id + AND migration.relay_host_id = pin.relay_host_id + AND migration.assignment_epoch = pin.assignment_epoch + WHERE pin.user_id = assignment.user_id + AND pin.relay_host_id = assignment.relay_host_id + AND pin.assignment_epoch = assignment.assignment_epoch + AND pin.target_cell_id = assignment.cell_id + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ) + OR ( + cell.enabled = 0 + AND ( + EXISTS ( + SELECT 1 FROM relay_cell_connection_limits limits + WHERE limits.cell_id = assignment.cell_id + ) + OR EXISTS ( + SELECT 1 FROM relay_post_drain_migration_pins pin + JOIN relay_assignment_migrations migration + ON migration.user_id = pin.user_id + AND migration.relay_host_id = pin.relay_host_id + AND migration.assignment_epoch = pin.assignment_epoch + WHERE pin.user_id = assignment.user_id + AND pin.relay_host_id = assignment.relay_host_id + AND pin.assignment_epoch = assignment.assignment_epoch + AND pin.target_cell_id = assignment.cell_id + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ) + AND attempt.completed_at IS NOT NULL + AND attempt.aborted_at IS NULL + AND committed.cell_incarnation = runtime.cell_incarnation + AND fence.cell_incarnation = committed.cell_incarnation + AND committed.attested_at >= runtime.last_heartbeat_at + AND committed.expires_at > ? + AND fence.expires_at > ? + ) + ) + ORDER BY assignment.user_id, assignment.relay_host_id LIMIT ?`, + [1, cutoff, this.now(), this.now(), limit] + ) + let moved = 0 + for (const row of rows) { + try { + const assignment = await this.assign({ + userId: text(row, 'user_id'), + relayHostId: text(row, 'relay_host_id') + }) + if (assignment.cellId !== text(row, 'cell_id')) moved++ + } catch (error) { + if (!(error instanceof Error && error.message === 'relay_capacity_exhausted')) throw error + } + } + return moved + } + + async configureCell( + cell: RelayCellConfig, + admission: boolean | CellAdmissionState + ): Promise { + const now = this.now() + const state = typeof admission === 'boolean' ? stateFromEnabled(admission) : admission + await this.database.transaction(async (transaction) => { + const cells = await transaction.queryLocked( + `SELECT * FROM relay_cells ORDER BY cell_id ASC` + ) + const current = cells.find((row) => text(row, 'cell_id') === cell.id) + if (current && integer(current, 'reserved_requests') > cell.capacityRequests) { + throw new Error('cell_capacity_below_reserved') + } + // Deploy automation owns tagged URLs; a restarted revision must not overwrite them. + await transaction.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + cell_url = excluded.cell_url, + enabled = excluded.enabled, + capacity_requests = excluded.capacity_requests, + updated_at = excluded.updated_at`, + [ + cell.id, + cell.url, + state === 'existing-only' ? 0 : 1, + cell.capacityRequests, + 0, + 0, + now, + now + ] + ) + await transaction.query( + `INSERT INTO relay_cell_regions (cell_id, region) VALUES (?, ?) + ON CONFLICT (cell_id) DO UPDATE SET region = excluded.region`, + [cell.id, cell.region ?? RELAY_DEFAULT_REGION] + ) + await ensureCellAdmission(transaction, cell.id, state, now) + await setCellAdmissionBeforeBoundary(transaction, cell.id, state, now) + if ( + cell.connectionHardCap !== undefined && + cell.connectionUnobservedBound !== undefined + ) { + const currentLimit = ( + await transaction.queryLocked( + `SELECT hard_cap, unobserved_bound FROM relay_cell_connection_limits + WHERE cell_id = ?`, + [cell.id] + ) + )[0] + const limitChanged = + currentLimit !== undefined && + (integer(currentLimit, 'hard_cap') !== cell.connectionHardCap || + integer(currentLimit, 'unobserved_bound') !== + cell.connectionUnobservedBound) + await transaction.query( + `INSERT INTO relay_cell_connection_limits + (cell_id, hard_cap, unobserved_bound, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + hard_cap = excluded.hard_cap, + unobserved_bound = excluded.unobserved_bound, + updated_at = excluded.updated_at`, + [cell.id, cell.connectionHardCap, cell.connectionUnobservedBound, now] + ) + if (limitChanged) { + await transaction.query( + `DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, + [cell.id] + ) + await transaction.query( + `DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, + [cell.id] + ) + } + } + }) + } + + async startActiveCellEvacuations( + sourceCellId: string, + targetCellId: string, + limit: number + ): Promise { + const rows = await this.database.query( + `SELECT user_id, relay_host_id FROM relay_assignments + WHERE cell_id = ? AND + (reserved_controls > 0 OR reserved_splices > 0 OR reserved_invites > 0 OR + pending_installs > 0 OR pending_confirmations > 0 OR migration_leases > 0) + ORDER BY user_id, relay_host_id LIMIT ?`, + [sourceCellId, limit] + ) + let started = 0 + for (const row of rows) { + const migration = await this.startEvacuation( + { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }, + targetCellId, + sourceCellId + ) + if (migration) started++ + } + return started + } + + async cellEvacuationCapacity( + sourceCellId: string, + targetCellId: string + ): Promise { + const cells = await this.database.query( + `SELECT * FROM relay_cells WHERE cell_id IN (?, ?) ORDER BY cell_id`, + [sourceCellId, targetCellId] + ) + if (!cells.some((row) => text(row, 'cell_id') === sourceCellId)) { + throw new Error('source_cell_not_found') + } + const target = cells.find((row) => text(row, 'cell_id') === targetCellId) + if (!target) throw new Error('target_cell_not_found') + if (!(await this.cellIsLive(this.database, targetCellId, this.now()))) { + throw new Error('target_cell_unavailable') + } + const summary = ( + await this.database.query( + `SELECT COUNT(*) AS assignments, + COALESCE(SUM((SELECT COALESCE(SUM(lease.request_units), 0) + FROM relay_assignment_activity_leases lease + WHERE lease.user_id = assignment.user_id + AND lease.relay_host_id = assignment.relay_host_id)), 0) AS source_units + FROM relay_assignments assignment + WHERE assignment.cell_id = ? AND + (assignment.reserved_controls > 0 OR assignment.reserved_splices > 0 OR + assignment.reserved_invites > 0 OR assignment.pending_installs > 0 OR + assignment.pending_confirmations > 0 OR assignment.migration_leases > 0)`, + [sourceCellId] + ) + )[0]! + const sourceAssignments = integer(summary, 'assignments') + return { + sourceAssignments, + // Each migration reserves its future target control plus all source-owned units. + requiredTargetUnits: integer(summary, 'source_units') + sourceAssignments, + availableTargetUnits: + integer(target, 'capacity_requests') - integer(target, 'reserved_requests') + } + } + + async cellEvacuationStatus( + sourceCellId: string, + targetCellId: string, + completeReady: boolean + ): Promise { + let completed = 0 + let blocked = 0 + const sourceFenced = await this.cellHasActiveFence(sourceCellId) + if (completeReady) { + const candidates = await this.activeCellMigrations(sourceCellId, targetCellId) + for (const [index, row] of candidates.entries()) { + try { + const identity = { + userId: text(row, 'user_id'), + relayHostId: text(row, 'relay_host_id') + } + const assignmentEpoch = integer(row, 'assignment_epoch') + if (sourceFenced) { + await this.completeEvacuationFromDeadSource(identity, { + assignmentEpoch, + sourceCellId, + targetCellId + }) + } else { + await this.completeEvacuation(identity, assignmentEpoch) + } + completed++ + } catch (error) { + if (error instanceof Error && error.message === 'migration_cell_inventory_busy') { + blocked += candidates.length - index + break + } + if (isIncompleteMigration(error)) blocked++ + else if (!(error instanceof Error && error.message === 'migration_not_found')) throw error + } + } + } + const active = await this.activeCellMigrations(sourceCellId, targetCellId) + const oldestExpiresAt = + active.length === 0 ? null : Math.min(...active.map((row) => integer(row, 'expires_at'))) + if (completeReady && active.length === 0) { + await this.reconcileReservationAccounting(sourceCellId, targetCellId) + } + const registered = active.filter( + (row) => optionalInteger(row, 'target_registered_at') !== undefined + ) + const registeredSourceActive = registered.filter( + (row) => integer(row, 'source_activity_units') > 0 + ).length + const registeredCompletable = registered.filter( + (row) => + integer(row, 'source_activity_units') === 0 && + integer(row, 'target_control_current') === 1 + ).length + const registeredTargetInactive = registered.filter( + (row) => + integer(row, 'source_activity_units') === 0 && + integer(row, 'target_control_current') === 0 + ).length + const expiredUnregistered = active.filter( + (row) => + optionalInteger(row, 'target_registered_at') === undefined && + integer(row, 'expires_at') <= this.now() + ) + const repairableExpiredUnregistered = expiredUnregistered.filter( + (row) => migrationHasExactActiveTarget(row, targetCellId) + ).length + const abortableExpiredUnregistered = expiredUnregistered.filter( + (row) => integer(row, 'target_control_active') === 0 + ).length + const blockedExpiredUnregistered = + expiredUnregistered.length - + repairableExpiredUnregistered - + abortableExpiredUnregistered + return { + inProgress: active.length, + oldestExpiresAt, + oldestRemainingMs: oldestExpiresAt === null ? null : oldestExpiresAt - this.now(), + targetRegistered: registered.length, + registeredSourceActive, + registeredCompletable, + registeredTargetInactive, + completed, + blocked, + expiredUnregistered: expiredUnregistered.length, + repairableExpiredUnregistered, + abortableExpiredUnregistered, + blockedExpiredUnregistered, + blockedExpiredOnNewerTargetAssignment: expiredUnregistered.filter( + (row) => + integer(row, 'target_control_active') === 1 && + optionalText(row, 'current_cell_id') === targetCellId && + (optionalInteger(row, 'current_assignment_epoch') ?? 0) > + integer(row, 'assignment_epoch') + ).length + } + } + + async completeReadyEvacuations(limit = 100): Promise { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) { + throw new Error('invalid_evacuation_completion_limit') + } + const now = this.now() + const runtimeSafety = this.requireLiveCells + ? `AND EXISTS ( + SELECT 1 FROM relay_cells source_cell + WHERE source_cell.cell_id = migration.source_cell_id + AND source_cell.enabled = 0 + ) + AND EXISTS ( + SELECT 1 FROM relay_cells target_cell + WHERE target_cell.cell_id = migration.target_cell_id + AND target_cell.enabled = 1 + ) + AND EXISTS ( + SELECT 1 FROM relay_cell_runtime source_runtime + WHERE source_runtime.cell_id = migration.source_cell_id + AND source_runtime.ready = 1 + AND source_runtime.observed_requests = 0 + AND source_runtime.last_heartbeat_at > ? + ) + AND EXISTS ( + SELECT 1 FROM relay_cell_runtime target_runtime + WHERE target_runtime.cell_id = migration.target_cell_id + AND target_runtime.ready = 1 + AND target_runtime.last_heartbeat_at > ? + )` + : '' + const targetIncarnation = this.requireLiveCells + ? `AND target_control.updated_at >= ( + SELECT target_runtime.started_at FROM relay_cell_runtime target_runtime + WHERE target_runtime.cell_id = migration.target_cell_id + )` + : '' + // Selection avoids polling offline desktops; completeEvacuation rechecks + // current target activity and zero source ownership under row locks. + const candidates = await this.database.query( + `SELECT migration.user_id, migration.relay_host_id, migration.source_cell_id, + migration.target_cell_id, migration.assignment_epoch + FROM relay_assignment_migrations migration + WHERE migration.target_registered_at IS NOT NULL + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + ${runtimeSafety} + AND NOT EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases source_lease + WHERE source_lease.user_id = migration.user_id + AND source_lease.relay_host_id = migration.relay_host_id + AND source_lease.cell_id = migration.source_cell_id + ) + AND EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases target_control + WHERE target_control.user_id = migration.user_id + AND target_control.relay_host_id = migration.relay_host_id + AND target_control.cell_id = migration.target_cell_id + AND target_control.activity_kind = 'control' + AND target_control.activity_id NOT LIKE 'control-pending:%' + AND target_control.expires_at > ? + ${targetIncarnation} + ) + ORDER BY migration.user_id, migration.relay_host_id + LIMIT ?`, + [ + ...(this.requireLiveCells + ? [now - this.heartbeatTtlMs, now - this.heartbeatTtlMs] + : []), + now, + limit + ] + ) + const pairs = new Map() + let completed = 0 + for (const row of candidates) { + const sourceCellId = text(row, 'source_cell_id') + const targetCellId = text(row, 'target_cell_id') + pairs.set(JSON.stringify([sourceCellId, targetCellId]), { sourceCellId, targetCellId }) + try { + await this.completeEvacuation( + { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }, + integer(row, 'assignment_epoch') + ) + completed++ + } catch (error) { + if (!isIncompleteMigration(error) && !isMissingMigration(error)) throw error + } + } + for (const { sourceCellId, targetCellId } of pairs.values()) { + if ((await this.activeCellMigrations(sourceCellId, targetCellId)).length === 0) { + await this.reconcileReservationAccounting(sourceCellId, targetCellId) + } + } + return completed + } + + async cellDeploymentStatus(cellId: string): Promise { + const cellRow = ( + await this.database.query( + `WITH activity AS ( + SELECT COUNT(*) AS activity_lease_count, + COALESCE(SUM(request_units), 0) AS activity_request_units, + COALESCE(SUM(CASE WHEN activity_kind = 'control' + AND activity_id LIKE 'control-pending:%' + AND request_units = 1 + THEN 1 ELSE 0 END), 0) AS pending_control_units + FROM relay_assignment_activity_leases WHERE cell_id = ? + ) + SELECT cell.*, runtime.cell_url AS runtime_cell_url, + admission.admission_state, + runtime.cell_incarnation AS runtime_cell_incarnation, + runtime.started_at AS runtime_started_at, runtime.ready AS runtime_ready, + runtime.observed_requests AS runtime_observed_requests, + runtime.last_heartbeat_at AS runtime_last_heartbeat_at, + COALESCE(capabilities.regional_rehome_protocol, 0) + AS runtime_regional_rehome_protocol, + activity.activity_lease_count, activity.activity_request_units, + activity.activity_lease_count - activity.pending_control_units + AS restart_blocking_activity_leases, + activity.activity_request_units - activity.pending_control_units + AS restart_blocking_activity_request_units, + cell.reserved_requests - activity.pending_control_units + AS restart_blocking_reserved_requests + FROM relay_cells cell + CROSS JOIN activity + LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id + LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id + LEFT JOIN relay_cell_capabilities capabilities + ON capabilities.cell_id = runtime.cell_id + AND capabilities.cell_incarnation = runtime.cell_incarnation + WHERE cell.cell_id = ?`, + [cellId, cellId] + ) + )[0] + if (!cellRow) throw new Error('cell_not_found') + const assignmentRow = ( + await this.database.query( + `SELECT COUNT(*) AS assignment_count FROM relay_assignments WHERE cell_id = ?`, + [cellId] + ) + )[0]! + const migrationRow = ( + await this.database.query( + `SELECT + COALESCE(SUM(CASE WHEN source_cell_id = ? THEN 1 ELSE 0 END), 0) + AS outgoing_migrations, + COALESCE(SUM(CASE WHEN target_cell_id = ? THEN 1 ELSE 0 END), 0) + AS incoming_migrations + FROM relay_assignment_migrations + WHERE completed_at IS NULL AND aborted_at IS NULL + AND (source_cell_id = ? OR target_cell_id = ?)`, + [cellId, cellId, cellId, cellId] + ) + )[0]! + const connectionRow = ( + await this.database.query( + `SELECT limits.hard_cap, limits.unobserved_bound, + connection_runtime.total_connections, connection_runtime.in_flight_connections, + connection_runtime.reserved_connection_units, + connection_runtime.enforced_connection_units, + connection_runtime.last_heartbeat_at, + current_runtime.cell_incarnation AS current_incarnation, + connection_runtime.cell_incarnation AS connection_incarnation, + (SELECT COUNT(*) FROM relay_control_connection_reservations reservation + WHERE reservation.cell_id = limits.cell_id + AND reservation.state IN + ('reserved', 'late-arrival-debt', 'claimed')) AS outstanding_reservations + FROM relay_cell_connection_limits limits + LEFT JOIN relay_cell_connection_runtime connection_runtime + ON connection_runtime.cell_id = limits.cell_id + LEFT JOIN relay_cell_runtime current_runtime + ON current_runtime.cell_id = limits.cell_id + WHERE limits.cell_id = ?`, + [cellId] + ) + )[0] + const runtimeHeartbeat = optionalInteger(cellRow, 'runtime_last_heartbeat_at') + const connectionHeartbeat = connectionRow + ? optionalInteger(connectionRow, 'last_heartbeat_at') + : undefined + return { + cellId, + cellUrl: text(cellRow, 'cell_url'), + region: await this.cellRegion(this.database, cellId), + enabled: integer(cellRow, 'enabled') === 1, + admissionState: parseCellAdmissionState(text(cellRow, 'admission_state')), + capacityRequests: integer(cellRow, 'capacity_requests'), + reservedRequests: integer(cellRow, 'reserved_requests'), + assignments: integer(assignmentRow, 'assignment_count'), + activityLeases: integer(cellRow, 'activity_lease_count'), + activityRequestUnits: integer(cellRow, 'activity_request_units'), + restartBlockingActivityLeases: integer( + cellRow, + 'restart_blocking_activity_leases' + ), + restartBlockingActivityRequestUnits: integer( + cellRow, + 'restart_blocking_activity_request_units' + ), + restartBlockingReservedRequests: integer( + cellRow, + 'restart_blocking_reserved_requests' + ), + outgoingMigrations: integer(migrationRow, 'outgoing_migrations'), + incomingMigrations: integer(migrationRow, 'incoming_migrations'), + connectionCapacity: connectionRow + ? { + hardCap: integer(connectionRow, 'hard_cap'), + controlRebindReserve: RELAY_ADMISSION_BUDGETS.reservedHostControls, + ordinaryConnectionLimit: + integer(connectionRow, 'hard_cap') - + RELAY_ADMISSION_BUDGETS.reservedHostControls, + unobservedBound: integer(connectionRow, 'unobserved_bound'), + normalAdmissionPause: + integer(connectionRow, 'hard_cap') - + RELAY_ADMISSION_BUDGETS.reservedHostControls - + integer(connectionRow, 'unobserved_bound'), + observedConnections: + optionalInteger(connectionRow, 'total_connections') ?? 0, + inFlightConnections: + optionalInteger(connectionRow, 'in_flight_connections') ?? 0, + reservedConnectionUnits: + optionalInteger(connectionRow, 'reserved_connection_units') ?? 0, + enforcedConnectionUnits: + optionalInteger(connectionRow, 'enforced_connection_units') ?? 0, + pendingControlReservations: integer( + connectionRow, + 'outstanding_reservations' + ), + heartbeatFresh: + connectionHeartbeat !== undefined && + connectionHeartbeat > this.now() - this.heartbeatTtlMs && + optionalText(connectionRow, 'current_incarnation') === + optionalText(connectionRow, 'connection_incarnation') + } + : null, + runtime: + runtimeHeartbeat === undefined + ? null + : { + cellUrl: text(cellRow, 'runtime_cell_url'), + cellIncarnation: text(cellRow, 'runtime_cell_incarnation'), + startedAt: integer(cellRow, 'runtime_started_at'), + ready: integer(cellRow, 'runtime_ready') === 1, + observedRequests: integer(cellRow, 'runtime_observed_requests'), + lastHeartbeatAt: runtimeHeartbeat, + heartbeatFresh: runtimeHeartbeat > this.now() - this.heartbeatTtlMs, + regionalRehomeProtocol: integer( + cellRow, + 'runtime_regional_rehome_protocol' + ) + } + } + } + + async verifyCellAssignment(input: AssignmentIdentity & { + cellId: string + assignmentEpoch: number + }): Promise { + const rows = await this.database.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [input.userId, input.relayHostId] + ) + return Boolean( + rows[0] && + text(rows[0], 'cell_id') === input.cellId && + integer(rows[0], 'assignment_epoch') === input.assignmentEpoch + ) + } + + async changeActivity( + identity: AssignmentIdentity, + kind: AssignmentActivityKind, + delta: 1 | -1 + ): Promise { + await this.activityQueue.run(identity, async () => { + const column = ACTIVITY_COLUMN[kind] + const now = this.now() + await this.database.transaction(async (transaction) => { + const row = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0] + if (!row) return + const before = integer(row, column) + const after = Math.max(0, before + delta) + await transaction.query( + `UPDATE relay_assignments SET ${column} = ?, lease_expires_at = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [after, now + ASSIGNMENT_LIMITS.activityLeaseMs, now, identity.userId, identity.relayHostId] + ) + const requestDelta = ACTIVITY_REQUEST_UNITS[kind] * (after - before) + if (requestDelta !== 0) { + await this.lockCellInventory(transaction) + await this.adjustCellReservation(transaction, text(row, 'cell_id'), requestDelta) + } + }) + }) + } + + async acquireActivity( + identity: AssignmentIdentity, + input: { + activityId: string + kind: AssignmentActivityKind + cellId: string + expiresAt?: number + } + ): Promise { + validateActivityId(input.activityId) + await this.activityQueue.run(identity, async () => { + const now = this.now() + await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + if (!assignment) throw new Error('assignment_not_found') + const assignmentCellId = text(assignment, 'cell_id') + if (input.cellId !== assignmentCellId) { + // Origin controls may renew work only while the exact forward + // migration is active; completion must fence late source activity. + const migration = ( + await transaction.queryLocked( + `SELECT assignment_epoch FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? + AND source_cell_id = ? AND target_cell_id = ? + AND assignment_epoch = ? + AND completed_at IS NULL AND aborted_at IS NULL`, + [ + identity.userId, + identity.relayHostId, + input.cellId, + assignmentCellId, + integer(assignment, 'assignment_epoch') + ] + ) + )[0] + if (!migration) throw new Error('activity_cell_not_authoritative') + } + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + const existing = activityLeaseById(activityLeases, input.activityId) + const expiresAt = input.expiresAt ?? now + ASSIGNMENT_LIMITS.activityLeaseMs + if (!Number.isSafeInteger(expiresAt) || expiresAt <= now) { + throw new Error('invalid_activity_expiry') + } + if (existing && text(existing, 'activity_kind') === input.kind && text(existing, 'cell_id') === input.cellId) { + await transaction.query( + `UPDATE relay_assignment_activity_leases SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [expiresAt, now, identity.userId, identity.relayHostId, input.activityId] + ) + await this.touchAssignment(transaction, identity, expiresAt, now) + return + } + const units = ACTIVITY_REQUEST_UNITS[input.kind] + if (existing) { + await this.lockCellInventory(transaction) + await this.removeActivityLease(transaction, identity, existing, now) + await this.adjustCellReservation(transaction, input.cellId, units) + } + await this.adjustActivityCount(transaction, identity, input.kind, 1, expiresAt, now) + await transaction.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + input.activityId, + input.kind, + input.cellId, + units, + expiresAt, + now + ] + ) + // Keep the contended cell row locked for only the final write and commit. + if (!existing) { + await this.adjustCellReservationAtomically(transaction, input.cellId, units) + } + }) + }) + } + + async renewControlActivity( + identity: AssignmentIdentity, + input: { activityId: string; cellId: string; expiresAt: number } + ): Promise { + validateActivityId(input.activityId) + const now = this.now() + const maximumExpiresAt = + now + + ASSIGNMENT_LIMITS.activityLeaseMs + + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2 + if ( + !Number.isSafeInteger(input.expiresAt) || + input.expiresAt <= now || + input.expiresAt > maximumExpiresAt + ) { + throw new Error('invalid_activity_expiry') + } + const startedAt = performance.now() + let outcome: ControlRenewalOutcome = 'database_error' + try { + outcome = + this.database.dialect === 'postgres' + ? await this.renewPostgresControlActivity(identity, input, now) + : await this.renewTransactionalControlActivity(identity, input, now) + if (outcome !== 'renewed') throw new Error(outcome) + } catch (error) { + const message = String((error as { message?: unknown }).message) + if (CONTROL_RENEWAL_OUTCOMES.has(message as ControlRenewalOutcome)) { + outcome = message as ControlRenewalOutcome + } + throw error + } finally { + this.recordControlRenewal?.(performance.now() - startedAt, outcome) + } + } + + private async renewPostgresControlActivity( + identity: AssignmentIdentity, + input: { activityId: string; cellId: string; expiresAt: number }, + now: number + ): Promise { + const row = ( + await this.database.query( + `WITH assignment_state AS MATERIALIZED ( + SELECT cell_id, assignment_epoch + FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ? + FOR UPDATE + ), migration_state AS MATERIALIZED ( + SELECT migration.assignment_epoch + FROM relay_assignment_migrations migration + JOIN assignment_state assignment + ON migration.target_cell_id = assignment.cell_id + AND migration.assignment_epoch = assignment.assignment_epoch + WHERE migration.user_id = ? AND migration.relay_host_id = ? + AND migration.source_cell_id = ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + FOR UPDATE OF migration + ), authorization_state AS MATERIALIZED ( + SELECT 1 AS authorized + FROM assignment_state assignment + WHERE assignment.cell_id = ? OR EXISTS (SELECT 1 FROM migration_state) + ), lease_state AS MATERIALIZED ( + SELECT lease.activity_kind, lease.cell_id + FROM relay_assignment_activity_leases lease + CROSS JOIN authorization_state + WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ? + FOR UPDATE OF lease + ), renewed_lease AS ( + UPDATE relay_assignment_activity_leases lease + SET expires_at = GREATEST(lease.expires_at, ?), + updated_at = GREATEST(lease.updated_at, ?) + FROM lease_state state + WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ? + AND state.activity_kind = 'control' AND state.cell_id = ? + RETURNING 1 + ), renewed_assignment AS ( + UPDATE relay_assignments assignment + SET lease_expires_at = GREATEST(assignment.lease_expires_at, ?), + last_activity_at = GREATEST(assignment.last_activity_at, ?) + WHERE assignment.user_id = ? AND assignment.relay_host_id = ? + AND EXISTS (SELECT 1 FROM renewed_lease) + RETURNING 1 + ) + SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM assignment_state) + THEN 'assignment_not_found' + WHEN NOT EXISTS (SELECT 1 FROM authorization_state) + THEN 'activity_cell_not_authoritative' + WHEN NOT EXISTS (SELECT 1 FROM lease_state) + THEN 'control_activity_not_found' + WHEN EXISTS ( + SELECT 1 FROM lease_state + WHERE activity_kind <> 'control' OR cell_id <> ? + ) THEN 'control_activity_moved' + WHEN EXISTS (SELECT 1 FROM renewed_assignment) THEN 'renewed' + ELSE 'control_activity_not_found' + END AS outcome`, + [ + identity.userId, + identity.relayHostId, + identity.userId, + identity.relayHostId, + input.cellId, + input.cellId, + identity.userId, + identity.relayHostId, + input.activityId, + input.expiresAt, + now, + identity.userId, + identity.relayHostId, + input.activityId, + input.cellId, + input.expiresAt, + now, + identity.userId, + identity.relayHostId, + input.cellId + ] + ) + )[0] + if (!row) throw new Error('missing_control_renewal_outcome') + const outcome = text(row, 'outcome') as ControlRenewalOutcome + if (!CONTROL_RENEWAL_OUTCOMES.has(outcome)) throw new Error('invalid_control_renewal_outcome') + return outcome + } + + private async renewTransactionalControlActivity( + identity: AssignmentIdentity, + input: { activityId: string; cellId: string; expiresAt: number }, + now: number + ): Promise { + // Bypass the process queue so a network-stalled activity call cannot suppress renewal. + await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + if (!assignment) throw new Error('assignment_not_found') + const assignmentCellId = text(assignment, 'cell_id') + if (input.cellId !== assignmentCellId) { + const migration = ( + await transaction.queryLocked( + `SELECT assignment_epoch FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? + AND source_cell_id = ? AND target_cell_id = ? + AND assignment_epoch = ? + AND completed_at IS NULL AND aborted_at IS NULL`, + [ + identity.userId, + identity.relayHostId, + input.cellId, + assignmentCellId, + integer(assignment, 'assignment_epoch') + ] + ) + )[0] + if (!migration) throw new Error('activity_cell_not_authoritative') + } + const lease = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, input.activityId] + ) + )[0] + if (!lease) throw new Error('control_activity_not_found') + if ( + text(lease, 'activity_kind') !== 'control' || + text(lease, 'cell_id') !== input.cellId + ) { + throw new Error('control_activity_moved') + } + await transaction.query( + `UPDATE relay_assignment_activity_leases SET + expires_at = CASE WHEN expires_at > ? THEN expires_at ELSE ? END, + updated_at = CASE WHEN updated_at > ? THEN updated_at ELSE ? END + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [ + input.expiresAt, + input.expiresAt, + now, + now, + identity.userId, + identity.relayHostId, + input.activityId + ] + ) + await transaction.query( + `UPDATE relay_assignments SET + lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, + last_activity_at = CASE WHEN last_activity_at > ? THEN last_activity_at ELSE ? END + WHERE user_id = ? AND relay_host_id = ?`, + [input.expiresAt, input.expiresAt, now, now, identity.userId, identity.relayHostId] + ) + }) + return 'renewed' + } + + async releaseActivity(identity: AssignmentIdentity, activityId: string): Promise { + validateActivityId(activityId) + return await this.activityQueue.run(identity, async () => { + const now = this.now() + return await this.database.transaction(async (transaction) => { + await this.assignmentRow(transaction, identity) + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + const existing = activityLeaseById(activityLeases, activityId) + if (!existing) return false + await transaction.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, activityId] + ) + await this.adjustActivityCount( + transaction, + identity, + activityKind(existing), + -1, + now, + now + ) + // Release paths can safely defer the cell-row lock until their final write. + await this.adjustCellReservationAtomically( + transaction, + text(existing, 'cell_id'), + -integer(existing, 'request_units') + ) + return true + }) + }) + } + + async activateControl( + identity: AssignmentIdentity, + input: { + cellId: string + assignmentEpoch: number + generation: number + connectionInclusionWatermark?: number + } + ): Promise { + const activityId = `control:${input.cellId}:${input.generation}` + validateActivityId(activityId) + return await this.activityQueue.run(identity, async () => { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + if ( + !assignment || + text(assignment, 'cell_id') !== input.cellId || + integer(assignment, 'assignment_epoch') !== input.assignmentEpoch + ) { + throw new Error('wrong_assignment') + } + const expiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + const existing = activityLeaseById(activityLeases, activityId) + const pendingId = pendingControlActivityId(input.assignmentEpoch) + const pending = activityLeaseById(activityLeases, pendingId) + const retainedActivityId = + existing || (pending && text(pending, 'cell_id') === input.cellId) + ? existing + ? activityId + : pendingId + : activityId + await this.removeSupersededSameCellControls( + transaction, + identity, + activityLeases, + input.cellId, + retainedActivityId, + now + ) + if (existing) { + await transaction.query( + `UPDATE relay_assignment_activity_leases SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [expiresAt, now, identity.userId, identity.relayHostId, activityId] + ) + await this.touchAssignment(transaction, identity, expiresAt, now) + } else { + if (pending && text(pending, 'cell_id') === input.cellId) { + await transaction.query( + `UPDATE relay_assignment_activity_leases + SET activity_id = ?, expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [activityId, expiresAt, now, identity.userId, identity.relayHostId, pendingId] + ) + await this.touchAssignment(transaction, identity, expiresAt, now) + } else { + await this.lockCellInventory(transaction) + await this.adjustCellReservation(transaction, input.cellId, 1) + await this.adjustActivityCount(transaction, identity, 'control', 1, expiresAt, now) + await transaction.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + activityId, + 'control', + input.cellId, + 1, + expiresAt, + now + ] + ) + } + } + await this.claimControlConnectionReservation( + transaction, + identity, + input.cellId, + input.assignmentEpoch, + activityId, + input.connectionInclusionWatermark, + now + ) + return activityId + }) + }) + } + + async startEvacuation( + identity: AssignmentIdentity, + targetCellId: string + ): Promise + async startEvacuation( + identity: AssignmentIdentity, + targetCellId: string, + expectedSourceCellId: string + ): Promise + async startEvacuation( + identity: AssignmentIdentity, + targetCellId: string, + expectedSourceCellId?: string + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + if (!assignment) throw new Error('assignment_not_found') + const sourceCellId = text(assignment, 'cell_id') + // Bulk selection is intentionally staleable; fence it before considering + // an existing migration that already moved the assignment to its target. + if (expectedSourceCellId && sourceCellId !== expectedSourceCellId) return null + const existing = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND completed_at IS NULL + AND aborted_at IS NULL AND expires_at > ? + ORDER BY assignment_epoch DESC LIMIT 1`, + [identity.userId, identity.relayHostId, now] + ) + )[0] + if (existing) { + if (text(existing, 'target_cell_id') !== targetCellId) { + throw new Error('migration_in_progress') + } + return migration(identity, existing) + } + if (sourceCellId === targetCellId) throw new Error('target_matches_source') + await this.lockAssignmentActivities(transaction, identity) + const cells = await this.lockCellInventory(transaction) + const target = cells.find((row) => text(row, 'cell_id') === targetCellId) + if (!target || integer(target, 'enabled') !== 1) throw new Error('target_cell_unavailable') + if (!(await this.cellIsLive(transaction, targetCellId, now))) { + throw new Error('target_cell_unavailable') + } + let sourceCellIncarnation: string | undefined + let targetCellIncarnation: string | undefined + if (this.requireLiveCells) { + const runtimes = await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime WHERE cell_id IN (?, ?) ORDER BY cell_id`, + [sourceCellId, targetCellId] + ) + const sourceRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === sourceCellId + ) + const targetRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === targetCellId + ) + if ( + !sourceRuntime || + integer(sourceRuntime, 'ready') !== 1 || + integer(sourceRuntime, 'last_heartbeat_at') <= now - this.heartbeatTtlMs + ) { + throw new Error('source_cell_unavailable') + } + if ( + !targetRuntime || + integer(targetRuntime, 'ready') !== 1 || + integer(targetRuntime, 'last_heartbeat_at') <= now - this.heartbeatTtlMs + ) { + throw new Error('target_cell_unavailable') + } + sourceCellIncarnation = text(sourceRuntime, 'cell_incarnation') + targetCellIncarnation = text(targetRuntime, 'cell_incarnation') + } + await this.assertCellConnectionHeadroom(transaction, targetCellId) + const sourceUnitsRow = ( + await transaction.query( + `SELECT COALESCE(SUM(request_units), 0) AS units + FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND cell_id = ?`, + [identity.userId, identity.relayHostId, sourceCellId] + ) + )[0] + const sourceRequestUnits = integer(sourceUnitsRow!, 'units') + const targetReservedUnits = sourceRequestUnits + 1 + await this.adjustCellReservation(transaction, targetCellId, targetReservedUnits) + const previousEpoch = integer(assignment, 'assignment_epoch') + const assignmentEpoch = previousEpoch + 1 + const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs + await transaction.query( + `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, + reserved_controls = reserved_controls + 1, + migration_leases = migration_leases + 1, + lease_expires_at = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + targetCellId, + assignmentEpoch, + expiresAt, + now, + identity.userId, + identity.relayHostId + ] + ) + await transaction.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + 'control', + targetCellId, + 1, + expiresAt, + now, + identity.userId, + identity.relayHostId, + migrationActivityId(assignmentEpoch), + 'migration', + targetCellId, + sourceRequestUnits, + expiresAt, + now + ] + ) + await this.insertControlConnectionReservation( + transaction, + identity, + targetCellId, + assignmentEpoch, + expiresAt, + now + ) + await transaction.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + sourceCellId, + targetCellId, + previousEpoch, + assignmentEpoch, + sourceRequestUnits, + targetReservedUnits, + expiresAt, + now, + now + ] + ) + if (sourceCellIncarnation && targetCellIncarnation) { + await transaction.query( + `INSERT INTO relay_assignment_migration_incarnations + (user_id, relay_host_id, assignment_epoch, source_cell_incarnation, + target_cell_incarnation) + VALUES (?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + sourceCellIncarnation, + targetCellIncarnation + ] + ) + } + return { + ...identity, + sourceCellId, + targetCellId, + previousEpoch, + assignmentEpoch, + expiresAt + } + }) + } + + async markMigrationTargetRegistered( + identity: AssignmentIdentity, + input: { cellId: string; assignmentEpoch: number } + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const rows = await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND target_cell_id = ? AND completed_at IS NULL AND aborted_at IS NULL`, + [ + identity.userId, + identity.relayHostId, + input.assignmentEpoch, + input.cellId + ] + ) + if (!rows[0]) return false + await transaction.query( + `UPDATE relay_assignment_migrations + SET target_registered_at = COALESCE(target_registered_at, ?), updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + return true + }) + } + + async completeEvacuationFromDeadSource( + identity: AssignmentIdentity, + input: { + assignmentEpoch: number + sourceCellId: string + targetCellId: string + } + ): Promise { + try { + return await this.withAssignmentLockRetry( + async (inventoryFirst) => + await this.completeEvacuationFromDeadSourceOnce(identity, input, inventoryFirst) + ) + } catch (error) { + if (isDatabaseLockUnavailable(error)) { + throw new Error('migration_cell_inventory_busy') + } + throw error + } + } + + private async completeEvacuationFromDeadSourceOnce( + identity: AssignmentIdentity, + input: { + assignmentEpoch: number + sourceCellId: string + targetCellId: string + }, + inventoryFirst: boolean + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + let lockedCells: SqlRow[] | undefined + if (inventoryFirst) { + try { + lockedCells = await this.lockCellInventory(transaction) + } catch (error) { + if (isDatabaseLockTimeout(error)) { + throw new Error('database_lock_unavailable') + } + throw error + } + } + const assignment = await this.assignmentRow(transaction, identity, inventoryFirst) + const row = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + )[0] + if (!row) throw new Error('migration_not_found') + assertMigrationPair(row, input.sourceCellId, input.targetCellId) + if (optionalInteger(row, 'completed_at') !== undefined) { + return deadSourceCompletionResult(row, false) + } + if (optionalInteger(row, 'aborted_at') !== undefined) { + throw new Error('migration_already_aborted') + } + if (optionalInteger(row, 'target_registered_at') === undefined) { + throw new Error('migration_target_not_registered') + } + assertCurrentMigrationAssignment(assignment, row) + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + assertAssignmentActivityAccounting(assignment, activityLeases, row) + if ( + activityLeases.some( + (lease) => + ![input.sourceCellId, input.targetCellId].includes(text(lease, 'cell_id')) + ) + ) { + throw new Error('migration_activity_topology_mismatch') + } + if (activityUnitsForCell(activityLeases, input.sourceCellId) > 0) { + throw new Error('migration_source_still_active') + } + const cells = lockedCells ?? (await this.lockCellInventory(transaction, true)) + const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId) + const target = cells.find((cell) => text(cell, 'cell_id') === input.targetCellId) + if (!source || integer(source, 'enabled') !== 0) { + throw new Error('migration_source_admission_changed') + } + if (!target || integer(target, 'enabled') !== 1) { + throw new Error('migration_target_admission_changed') + } + await assertCellReservationAccounting(transaction, cells, [ + input.sourceCellId, + input.targetCellId + ]) + const runtimes = await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime WHERE cell_id IN (?, ?) ORDER BY cell_id`, + [input.sourceCellId, input.targetCellId] + ) + const sourceRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === input.sourceCellId + ) + const targetRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === input.targetCellId + ) + const freshAfter = now - this.heartbeatTtlMs + await this.requireCellFence(transaction, input.sourceCellId, sourceRuntime, now) + if (!sourceRuntime || integer(sourceRuntime, 'last_heartbeat_at') > freshAfter) { + throw new Error('migration_source_runtime_not_dead') + } + if ( + !targetRuntime || + integer(targetRuntime, 'ready') !== 1 || + integer(targetRuntime, 'last_heartbeat_at') <= freshAfter + ) { + throw new Error('migration_target_runtime_not_ready') + } + const targetIsActive = activityLeases.some( + (lease) => + text(lease, 'cell_id') === input.targetCellId && + text(lease, 'activity_kind') === 'control' && + !text(lease, 'activity_id').startsWith('control-pending:') && + integer(lease, 'expires_at') > now && + integer(lease, 'updated_at') >= integer(targetRuntime, 'started_at') + ) + if (!targetIsActive) { + const inactiveTargetControls = activityLeases.filter( + (lease) => + text(lease, 'cell_id') === input.targetCellId && + text(lease, 'activity_kind') === 'control' + ) + for (const lease of inactiveTargetControls) { + await this.removeActivityLease(transaction, identity, lease, now) + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + input.targetCellId, + input.assignmentEpoch, + now + ) + } + const migrationLease = activityLeaseById( + activityLeases, + migrationActivityId(input.assignmentEpoch) + ) + if (migrationLease) { + await this.removeActivityLease(transaction, identity, migrationLease, now) + } + await transaction.query( + `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + return deadSourceCompletionResult(row, true) + }) + } + + async supersedeRegisteredEvacuation( + identity: AssignmentIdentity, + input: RegisteredEvacuationSupersessionInput, + preservedActivityCellIds: readonly string[] = [] + ): Promise { + if (input.assignmentEpoch >= Number.MAX_SAFE_INTEGER) { + throw new Error('assignment_epoch_exhausted') + } + return await this.withAssignmentLockRetry( + async (inventoryFirst) => + await this.supersedeRegisteredEvacuationOnce( + identity, + input, + inventoryFirst, + preservedActivityCellIds + ) + ) + } + + private async supersedeRegisteredEvacuationOnce( + identity: AssignmentIdentity, + input: RegisteredEvacuationSupersessionInput, + inventoryFirst: boolean, + preservedActivityCellIds: readonly string[] + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const lockedCells = inventoryFirst + ? await this.lockCellInventory(transaction) + : undefined + const assignment = await this.assignmentRow(transaction, identity, inventoryFirst) + const existing = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + )[0] + if (!existing) throw new Error('migration_not_found') + assertMigrationPair(existing, input.sourceCellId, input.currentTargetCellId) + if (optionalInteger(existing, 'completed_at') !== undefined) { + throw new Error('migration_already_completed') + } + if (optionalInteger(existing, 'aborted_at') !== undefined) { + const successor = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND previous_epoch = ? + AND source_cell_id = ? AND target_cell_id = ? + ORDER BY assignment_epoch DESC LIMIT 1`, + [ + identity.userId, + identity.relayHostId, + input.assignmentEpoch, + input.sourceCellId, + input.replacementTargetCellId + ] + ) + )[0] + if (!successor) throw new Error('migration_already_superseded') + return migration(identity, successor) + } + if (optionalInteger(existing, 'target_registered_at') === undefined) { + throw new Error('migration_target_not_registered') + } + const existingIncarnation = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migration_incarnations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + )[0] + const existingPin = ( + await transaction.queryLocked( + `SELECT * FROM relay_post_drain_migration_pins + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + )[0] + if (existingPin && !existingIncarnation) { + throw new Error('drain_migration_source_incarnation_mismatch') + } + assertCurrentMigrationAssignment(assignment, existing) + if (input.currentTargetCellId === input.replacementTargetCellId) { + throw new Error('target_matches_source') + } + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + assertAssignmentActivityAccounting(assignment, activityLeases, existing) + const additionalActivityCellIds = new Set( + activityLeases + .map((lease) => text(lease, 'cell_id')) + .filter( + (cellId) => + ![input.sourceCellId, input.currentTargetCellId].includes(cellId) + ) + ) + if ( + preservedActivityCellIds.includes(input.replacementTargetCellId) || + !matchesExactCellSet(additionalActivityCellIds, preservedActivityCellIds) + ) { + throw new Error('migration_activity_topology_mismatch') + } + const cells = lockedCells ?? (await this.lockCellInventory(transaction, true)) + const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId) + const currentTarget = cells.find( + (cell) => text(cell, 'cell_id') === input.currentTargetCellId + ) + const replacement = cells.find( + (cell) => text(cell, 'cell_id') === input.replacementTargetCellId + ) + if (!source || integer(source, 'enabled') !== 0) { + throw new Error('migration_source_admission_changed') + } + if (!currentTarget || integer(currentTarget, 'enabled') !== 0) { + throw new Error('migration_target_still_enabled') + } + if (!replacement || integer(replacement, 'enabled') !== 1) { + throw new Error('replacement_target_unavailable') + } + await assertCellReservationAccounting(transaction, cells, [ + input.sourceCellId, + input.currentTargetCellId, + input.replacementTargetCellId, + ...preservedActivityCellIds + ]) + const runtimeCellIds = [ + input.currentTargetCellId, + input.replacementTargetCellId, + ...preservedActivityCellIds + ] + const runtimes = await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime + WHERE cell_id IN (${runtimeCellIds.map(() => '?').join(', ')}) + ORDER BY cell_id`, + runtimeCellIds + ) + const currentRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === input.currentTargetCellId + ) + const replacementRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === input.replacementTargetCellId + ) + const freshAfter = now - this.heartbeatTtlMs + await this.requireCellFence( + transaction, + input.currentTargetCellId, + currentRuntime, + now + ) + if ( + currentRuntime && + integer(currentRuntime, 'ready') === 1 && + integer(currentRuntime, 'last_heartbeat_at') > freshAfter + ) { + throw new Error('migration_target_still_available') + } + if ( + !replacementRuntime || + integer(replacementRuntime, 'ready') !== 1 || + integer(replacementRuntime, 'last_heartbeat_at') <= freshAfter + ) { + throw new Error('replacement_target_unavailable') + } + if ( + preservedActivityCellIds.some((cellId) => { + const runtime = runtimes.find((row) => text(row, 'cell_id') === cellId) + return ( + !runtime || + integer(runtime, 'ready') !== 1 || + integer(runtime, 'last_heartbeat_at') <= freshAfter + ) + }) + ) { + throw new Error('migration_preserved_activity_cell_unavailable') + } + await this.assertCellConnectionHeadroom( + transaction, + input.replacementTargetCellId + ) + for (const lease of activityLeases.filter( + (candidate) => text(candidate, 'cell_id') === input.currentTargetCellId + )) { + await this.removeActivityLease(transaction, identity, lease, now) + } + const sourceRequestUnits = activityUnitsForCell(activityLeases, input.sourceCellId) + const targetReservedUnits = sourceRequestUnits + 1 + await this.adjustCellReservation( + transaction, + input.replacementTargetCellId, + targetReservedUnits + ) + const assignmentEpoch = input.assignmentEpoch + 1 + const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs + await transaction.query( + `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, + reserved_controls = reserved_controls + 1, + migration_leases = migration_leases + 1, + lease_expires_at = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + input.replacementTargetCellId, + assignmentEpoch, + expiresAt, + now, + identity.userId, + identity.relayHostId + ] + ) + await transaction.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + 'control', + input.replacementTargetCellId, + 1, + expiresAt, + now, + identity.userId, + identity.relayHostId, + migrationActivityId(assignmentEpoch), + 'migration', + input.replacementTargetCellId, + sourceRequestUnits, + expiresAt, + now + ] + ) + await this.insertControlConnectionReservation( + transaction, + identity, + input.replacementTargetCellId, + assignmentEpoch, + expiresAt, + now + ) + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + input.currentTargetCellId, + input.assignmentEpoch, + now + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + await transaction.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + input.sourceCellId, + input.replacementTargetCellId, + input.assignmentEpoch, + assignmentEpoch, + sourceRequestUnits, + targetReservedUnits, + expiresAt, + now, + now + ] + ) + if (existingIncarnation) { + await transaction.query( + `INSERT INTO relay_assignment_migration_incarnations + (user_id, relay_host_id, assignment_epoch, source_cell_incarnation, + target_cell_incarnation) + VALUES (?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + text(existingIncarnation, 'source_cell_incarnation'), + text(replacementRuntime, 'cell_incarnation') + ] + ) + } + if (existingPin) { + await transaction.query( + `INSERT INTO relay_post_drain_migration_pins + (user_id, relay_host_id, assignment_epoch, drain_attempt_id, + source_cell_id, source_cell_incarnation, target_cell_id, + target_cell_incarnation, source_request_units, target_reserved_units, + pinned_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + text(existingPin, 'drain_attempt_id'), + input.sourceCellId, + text(existingPin, 'source_cell_incarnation'), + input.replacementTargetCellId, + text(replacementRuntime, 'cell_incarnation'), + sourceRequestUnits, + targetReservedUnits, + now + ] + ) + } + return { + ...identity, + sourceCellId: input.sourceCellId, + targetCellId: input.replacementTargetCellId, + previousEpoch: input.assignmentEpoch, + assignmentEpoch, + expiresAt + } + }) + } + + async supersedeRegisteredCellEvacuations( + sourceCellId: string, + currentTargetCellId: string, + replacementTargetCellId: string, + limit: number + ): Promise { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) { + throw new Error('invalid_evacuation_limit') + } + const rows = await this.database.query( + `SELECT user_id, relay_host_id, assignment_epoch + FROM relay_assignment_migrations + WHERE source_cell_id = ? AND target_cell_id = ? + AND target_registered_at IS NOT NULL + AND completed_at IS NULL AND aborted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM relay_assignment_migrations earlier + WHERE earlier.user_id = relay_assignment_migrations.user_id + AND earlier.relay_host_id = relay_assignment_migrations.relay_host_id + AND earlier.source_cell_id = relay_assignment_migrations.source_cell_id + AND earlier.target_cell_id = relay_assignment_migrations.target_cell_id + AND earlier.target_registered_at IS NOT NULL + AND earlier.completed_at IS NULL AND earlier.aborted_at IS NULL + AND earlier.assignment_epoch < relay_assignment_migrations.assignment_epoch + ) + ORDER BY user_id, relay_host_id, assignment_epoch + LIMIT ?`, + [sourceCellId, currentTargetCellId, limit] + ) + let superseded = 0 + let reconciledAccounting = false + for (const row of rows) { + const identity = { + userId: text(row, 'user_id'), + relayHostId: text(row, 'relay_host_id') + } + const input = { + assignmentEpoch: integer(row, 'assignment_epoch'), + sourceCellId, + currentTargetCellId, + replacementTargetCellId + } + const prepared = await this.prepareRegisteredCellSupersession(identity, input) + if (prepared.retired) { + input.assignmentEpoch = integer(row, 'assignment_epoch') + await this.supersedeRegisteredEvacuation(identity, input) + superseded++ + continue + } + input.assignmentEpoch = prepared.assignmentEpoch + try { + await this.supersedeRegisteredEvacuation(identity, input) + } catch (error) { + if ( + error instanceof Error && + error.message === 'migration_activity_topology_mismatch' + ) { + const preservedActivityCellIds = await this.additionalActivityCellIds( + identity, + sourceCellId, + currentTargetCellId + ) + if ( + preservedActivityCellIds.length === 0 || + preservedActivityCellIds.includes(replacementTargetCellId) + ) { + throw error + } + await this.reconcileReservationAccounting(sourceCellId, currentTargetCellId) + await this.reconcileReservationAccounting(sourceCellId, replacementTargetCellId) + for (const cellId of preservedActivityCellIds) { + await this.reconcileReservationAccounting(sourceCellId, cellId) + } + await this.supersedeRegisteredEvacuation( + identity, + input, + preservedActivityCellIds + ) + superseded++ + continue + } + if ( + reconciledAccounting || + !(error instanceof Error) || + error.message !== 'migration_cell_reservation_accounting_mismatch' + ) { + throw error + } + await this.reconcileReservationAccounting(sourceCellId, currentTargetCellId) + await this.reconcileReservationAccounting(sourceCellId, replacementTargetCellId) + reconciledAccounting = true + await this.supersedeRegisteredEvacuation(identity, input) + } + superseded++ + } + return superseded + } + + private async prepareRegisteredCellSupersession( + identity: AssignmentIdentity, + input: RegisteredEvacuationSupersessionInput + ): Promise<{ assignmentEpoch: number; retired: boolean }> { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + const staleMigration = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + )[0] + if (!assignment || !staleMigration) throw new Error('migration_assignment_mismatch') + assertMigrationPair( + staleMigration, + input.sourceCellId, + input.currentTargetCellId + ) + if ( + optionalInteger(staleMigration, 'completed_at') !== undefined || + optionalInteger(staleMigration, 'aborted_at') !== undefined + ) { + return { assignmentEpoch: integer(assignment, 'assignment_epoch'), retired: true } + } + if (optionalInteger(staleMigration, 'target_registered_at') === undefined) { + throw new Error('migration_not_registered_for_supersession') + } + const assignmentEpoch = integer(assignment, 'assignment_epoch') + if (assignmentEpoch < input.assignmentEpoch) { + throw new Error('migration_assignment_mismatch') + } + const assignmentCellId = text(assignment, 'cell_id') + if ( + ![input.currentTargetCellId, input.replacementTargetCellId].includes( + assignmentCellId + ) + ) { + throw new Error('migration_assignment_mismatch') + } + const leases = await this.lockAssignmentActivities(transaction, identity) + const staleIncarnation = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migration_incarnations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + )[0] + const stalePin = ( + await transaction.queryLocked( + `SELECT * FROM relay_post_drain_migration_pins + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + )[0] + assertMigrationRecoveryMetadata(staleMigration, staleIncarnation, stalePin) + const failedRuntime = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, + [input.currentTargetCellId] + ) + )[0] + await this.requireCellFence( + transaction, + input.currentTargetCellId, + failedRuntime, + now + ) + if ( + assignmentEpoch > input.assignmentEpoch && + assignmentCellId === input.replacementTargetCellId + ) { + const obsoleteActivityIds = new Set([ + pendingControlActivityId(input.assignmentEpoch), + migrationActivityId(input.assignmentEpoch) + ]) + const obsoleteLeases = leases.filter((lease) => + obsoleteActivityIds.has(text(lease, 'activity_id')) + ) + assertAssignmentActivityCounts( + assignment, + leases, + leases.filter((lease) => activityKind(lease) === 'migration').length + ) + for (const lease of obsoleteLeases) { + const kind = activityKind(lease) + const expectedUnits = + kind === 'migration' + ? integer(staleMigration, 'source_request_units') + : ACTIVITY_REQUEST_UNITS.control + if ( + text(lease, 'cell_id') !== input.currentTargetCellId || + !['control', 'migration'].includes(kind) || + integer(lease, 'request_units') !== expectedUnits + ) { + throw new Error('migration_activity_topology_mismatch') + } + } + if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction) + for (const lease of obsoleteLeases) { + await this.removeActivityLease(transaction, identity, lease, now) + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + input.currentTargetCellId, + input.assignmentEpoch, + now + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + return { assignmentEpoch, retired: true } + } + let migrationRow = staleMigration + if (assignmentEpoch > input.assignmentEpoch) { + const existingCurrent = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + if (existingCurrent) { + assertMigrationPair( + existingCurrent, + input.sourceCellId, + input.currentTargetCellId + ) + if ( + optionalInteger(existingCurrent, 'completed_at') !== undefined || + optionalInteger(existingCurrent, 'aborted_at') !== undefined || + optionalInteger(existingCurrent, 'target_registered_at') === undefined + ) { + throw new Error('migration_activity_topology_mismatch') + } + assertCurrentMigrationAssignment(assignment, existingCurrent) + const currentIncarnation = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migration_incarnations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + const currentPin = ( + await transaction.queryLocked( + `SELECT * FROM relay_post_drain_migration_pins + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + assertMigrationRecoveryMetadata( + existingCurrent, + currentIncarnation, + currentPin + ) + migrationRow = existingCurrent + } else { + if (leases.length !== 0) { + throw new Error('migration_activity_topology_mismatch') + } + assertAssignmentActivityCounts(assignment, leases, 0) + const currentIncarnations = await transaction.queryLocked( + `SELECT assignment_epoch FROM relay_assignment_migration_incarnations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + const currentPins = await transaction.queryLocked( + `SELECT assignment_epoch FROM relay_post_drain_migration_pins + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + if (currentIncarnations.length > 0 || currentPins.length > 0) { + throw new Error('migration_activity_topology_mismatch') + } + await transaction.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 0, 1, ?, ?, NULL, NULL, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + input.sourceCellId, + input.currentTargetCellId, + input.assignmentEpoch, + assignmentEpoch, + now - 1, + now, + now, + now + ] + ) + migrationRow = { + ...staleMigration, + assignment_epoch: assignmentEpoch, + previous_epoch: input.assignmentEpoch, + source_request_units: 0, + target_reserved_units: 1, + expires_at: now - 1, + target_registered_at: now, + completed_at: null, + aborted_at: null + } + if (staleIncarnation) { + await transaction.query( + `INSERT INTO relay_assignment_migration_incarnations + (user_id, relay_host_id, assignment_epoch, + source_cell_incarnation, target_cell_incarnation) + VALUES (?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + text(staleIncarnation, 'source_cell_incarnation'), + text(staleIncarnation, 'target_cell_incarnation') + ] + ) + } + if (stalePin) { + await transaction.query( + `INSERT INTO relay_post_drain_migration_pins + (user_id, relay_host_id, assignment_epoch, drain_attempt_id, + source_cell_id, source_cell_incarnation, target_cell_id, + target_cell_incarnation, source_request_units, + target_reserved_units, pinned_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, 1, ?)`, + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + text(stalePin, 'drain_attempt_id'), + input.sourceCellId, + text(stalePin, 'source_cell_incarnation'), + input.currentTargetCellId, + text(stalePin, 'target_cell_incarnation'), + now + ] + ) + } + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + input.currentTargetCellId, + input.assignmentEpoch, + now + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, input.assignmentEpoch] + ) + } + const migrationLeases = leases.filter( + (lease) => activityKind(lease) === 'migration' + ) + if (migrationLeases.length > 0) { + assertAssignmentActivityAccounting(assignment, leases, migrationRow) + return { assignmentEpoch, retired: false } + } + assertAssignmentActivityCounts(assignment, leases, 0) + const sourceRequestUnits = integer(migrationRow, 'source_request_units') + if ( + integer(migrationRow, 'expires_at') > now || + sourceRequestUnits < 0 || + integer(migrationRow, 'target_reserved_units') !== sourceRequestUnits + 1 + ) { + throw new Error('migration_activity_lease_shape_mismatch') + } + await this.lockCellInventory(transaction) + await this.adjustCellReservation( + transaction, + input.currentTargetCellId, + sourceRequestUnits + ) + const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs + await transaction.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, 'migration', ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + migrationActivityId(assignmentEpoch), + input.currentTargetCellId, + sourceRequestUnits, + expiresAt, + now + ] + ) + await transaction.query( + `UPDATE relay_assignments SET migration_leases = migration_leases + 1, + lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, + last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [expiresAt, expiresAt, now, identity.userId, identity.relayHostId] + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return { assignmentEpoch, retired: false } + }) + } + + private async additionalActivityCellIds( + identity: AssignmentIdentity, + sourceCellId: string, + currentTargetCellId: string + ): Promise { + const rows = await this.database.query( + `SELECT DISTINCT cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? + AND cell_id NOT IN (?, ?) + ORDER BY cell_id`, + [identity.userId, identity.relayHostId, sourceCellId, currentTargetCellId] + ) + return rows.map((row) => text(row, 'cell_id')) + } + + async completeEvacuation( + identity: AssignmentIdentity, + assignmentEpoch: number + ): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + const row = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND completed_at IS NULL AND aborted_at IS NULL`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + if (!row) throw new Error('migration_not_found') + if (optionalInteger(row, 'target_registered_at') === undefined) { + throw new Error('migration_target_not_registered') + } + const sourceCellId = text(row, 'source_cell_id') + const targetCellId = text(row, 'target_cell_id') + if ( + !assignment || + text(assignment, 'cell_id') !== targetCellId || + integer(assignment, 'assignment_epoch') !== assignmentEpoch + ) { + throw new Error('migration_assignment_mismatch') + } + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + const sourceUnits = activityUnitsForCell(activityLeases, sourceCellId) + if (sourceUnits > 0) throw new Error('migration_source_still_active') + let cellsLocked = false + let targetStartedAt = 0 + if (this.requireLiveCells) { + let cells: SqlRow[] + try { + cells = await this.lockCellInventory(transaction, true) + } catch (error) { + if (isDatabaseLockUnavailable(error)) { + // Mixed-version workers may still hold a cell-first lock; defer + // instead of waiting long enough to form their legacy lock cycle. + throw new Error('migration_cell_inventory_busy') + } + throw error + } + cellsLocked = true + const source = cells.find((cell) => text(cell, 'cell_id') === sourceCellId) + const target = cells.find((cell) => text(cell, 'cell_id') === targetCellId) + if (!source || integer(source, 'enabled') !== 0) { + throw new Error('migration_source_admission_changed') + } + if (!target || integer(target, 'enabled') !== 1) { + throw new Error('migration_target_admission_changed') + } + const runtimes = await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime WHERE cell_id IN (?, ?) ORDER BY cell_id`, + [sourceCellId, targetCellId] + ) + const sourceRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === sourceCellId + ) + const targetRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === targetCellId + ) + const freshAfter = now - this.heartbeatTtlMs + if ( + !sourceRuntime || + integer(sourceRuntime, 'ready') !== 1 || + integer(sourceRuntime, 'observed_requests') !== 0 || + integer(sourceRuntime, 'last_heartbeat_at') <= freshAfter + ) { + throw new Error('migration_source_runtime_not_quiescent') + } + if ( + !targetRuntime || + integer(targetRuntime, 'ready') !== 1 || + integer(targetRuntime, 'last_heartbeat_at') <= freshAfter + ) { + throw new Error('migration_target_runtime_not_ready') + } + targetStartedAt = integer(targetRuntime, 'started_at') + } + const targetIsActive = activityLeases.some( + (lease) => + text(lease, 'cell_id') === targetCellId && + text(lease, 'activity_kind') === 'control' && + !text(lease, 'activity_id').startsWith('control-pending:') && + integer(lease, 'expires_at') > now && + integer(lease, 'updated_at') >= targetStartedAt + ) + if (!targetIsActive) throw new Error('migration_target_not_active') + const lease = activityLeaseById(activityLeases, migrationActivityId(assignmentEpoch)) + if (lease && !cellsLocked) await this.lockCellInventory(transaction) + if (lease) await this.removeActivityLease(transaction, identity, lease, now) + await transaction.query( + `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + }) + } + + async rebalanceDormant( + identity: AssignmentIdentity, + targetCellId: string + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + if (!assignment) throw new Error('assignment_not_found') + if (!mayNormallyReassign(activity(assignment), now)) throw new Error('assignment_active') + const sourceCellId = text(assignment, 'cell_id') + if (sourceCellId === targetCellId) throw new Error('target_matches_source') + await this.lockAssignmentActivities(transaction, identity) + const cells = await this.lockCellInventory(transaction) + const admission = await cellAdmissionStates(transaction) + const targetRow = cells.find( + (row) => + text(row, 'cell_id') === targetCellId && + admission.get(targetCellId) === 'general' + ) + if (!targetRow) throw new Error('target_cell_unavailable') + if (!(await this.cellIsLive(transaction, targetCellId, now))) { + throw new Error('target_cell_unavailable') + } + await this.assertCellConnectionHeadroom(transaction, targetCellId) + const target = cell(targetRow, await this.cellRegion(transaction, targetCellId)) + await this.adjustCellReservation(transaction, targetCellId, 1) + const assignmentEpoch = integer(assignment, 'assignment_epoch') + 1 + const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs + await transaction.query( + `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, + reserved_controls = 1, lease_expires_at = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + targetCellId, + assignmentEpoch, + leaseExpiresAt, + now, + identity.userId, + identity.relayHostId + ] + ) + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + sourceCellId, + assignmentEpoch - 1, + now + ) + await this.insertPendingControlLease( + transaction, + identity, + targetCellId, + assignmentEpoch, + now + ) + return { + ...identity, + ...target, + assignmentEpoch, + leaseExpiresAt + } + }) + } + + async inspectRegionalRehomeControl(): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + await this.initializeRegionalRehomeControl(transaction, now) + const row = ( + await transaction.query( + `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'` + ) + )[0]! + return regionalRehomeControl(row) + }) + } + + async applyRegionalRehomeControl(input: { + expectedGeneration: number + enabled: boolean + notBefore: number + ratePerMinute: number + preferenceMaxAgeMs: number + drainGraceMs: number + }): Promise { + if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) { + throw new Error('invalid_regional_rehome_generation') + } + if (!Number.isSafeInteger(input.notBefore) || input.notBefore < 0) { + throw new Error('invalid_regional_rehome_not_before') + } + if (!Number.isSafeInteger(input.ratePerMinute) || input.ratePerMinute < 1 || input.ratePerMinute > 120) { + throw new Error('invalid_regional_rehome_rate') + } + if ( + !Number.isSafeInteger(input.preferenceMaxAgeMs) || + input.preferenceMaxAgeMs < 60_000 || + input.preferenceMaxAgeMs > 30 * 24 * 60 * 60_000 + ) { + throw new Error('invalid_regional_rehome_preference_age') + } + if ( + !Number.isSafeInteger(input.drainGraceMs) || + input.drainGraceMs < 60_000 || + input.drainGraceMs > 60 * 60_000 + ) { + throw new Error('invalid_regional_rehome_drain_grace') + } + const now = this.now() + return await this.database.transaction(async (transaction) => { + await this.initializeRegionalRehomeControl(transaction, now) + const current = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'` + ) + )[0]! + if (integer(current, 'generation') !== input.expectedGeneration) { + throw new Error('regional_rehome_generation_mismatch') + } + if ( + input.enabled && + input.notBefore < + integer(current, 'observation_started_at') + REGIONAL_REHOME_OBSERVATION_MS + ) { + throw new Error('regional_rehome_observation_window_incomplete') + } + await transaction.query( + `UPDATE relay_region_rehome_control + SET generation = generation + 1, enabled = ?, not_before = ?, + rate_per_minute = ?, preference_max_age_ms = ?, drain_grace_ms = ?, + updated_at = ? + WHERE control_id = 'global'`, + [ + input.enabled ? 1 : 0, + input.notBefore, + input.ratePerMinute, + input.preferenceMaxAgeMs, + input.drainGraceMs, + now + ] + ) + const updated = ( + await transaction.query( + `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'` + ) + )[0]! + return regionalRehomeControl(updated) + }) + } + + async disableRegionalRehomeControl(): Promise { + const now = this.now() + const result = await this.database.query( + `UPDATE relay_region_rehome_control + SET generation = generation + 1, enabled = 0, updated_at = ? + WHERE control_id = 'global' AND enabled = 1`, + [now] + ) + return integer(result[0]!, 'changes') === 1 + } + + private async initializeRegionalRehomeControl( + database: RelayDatabase, + now: number + ): Promise { + await database.query( + `INSERT INTO relay_region_rehome_control + (control_id, generation, enabled, observation_started_at, not_before, + rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at) + VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?) + ON CONFLICT (control_id) DO NOTHING`, + [now, 24 * 60 * 60_000, 60 * 60_000, now] + ) + } + + async regionalRehomeFleetSafety(): Promise { + return await this.readRegionalRehomeFleetSafety(this.database, this.now()) + } + + private async readRegionalRehomeFleetSafety( + database: RelayDatabase, + now: number + ): Promise { + const rows = await database.query( + `SELECT runtime.ready, runtime.last_heartbeat_at, + safety.cell_id AS safety_cell_id, safety.observed_at, + safety.sql_failures, safety.reconnects, + safety.control_activity_recovery_failures, + safety.database_pool_waiting, safety.database_pool_waiters_max, + safety.database_pool_wait_ms_max + FROM relay_cells cell + JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id + JOIN relay_cell_regions region ON region.cell_id = cell.cell_id + LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id + LEFT JOIN relay_cell_capabilities capability ON capability.cell_id = cell.cell_id + LEFT JOIN relay_cell_rehome_safety safety + ON safety.cell_id = runtime.cell_id + AND safety.cell_incarnation = runtime.cell_incarnation + WHERE cell.enabled = 1 AND admission.admission_state = 'general' + AND ( + region.region = 'asia-east2' OR + (region.region = 'us-central1' AND capability.regional_rehome_protocol >= 1) + )` + ) + const valid = rows.filter( + (row) => + integer(row, 'ready') === 1 && + integer(row, 'last_heartbeat_at') > now - this.heartbeatTtlMs && + optionalText(row, 'safety_cell_id') !== undefined && + integer(row, 'observed_at') > now - 60_000 + ) + const missingCells = rows.length === 0 ? 1 : rows.length - valid.length + return { + requiredCells: rows.length, + missingCells, + observedAt: + missingCells > 0 + ? 0 + : Math.min(...valid.map((row) => integer(row, 'observed_at'))), + sqlFailures: valid.reduce((total, row) => total + integer(row, 'sql_failures'), 0), + reconnects: valid.reduce((total, row) => total + integer(row, 'reconnects'), 0), + maxReconnects: Math.max(0, ...valid.map((row) => integer(row, 'reconnects'))), + controlActivityRecoveryFailures: valid.reduce( + (total, row) => total + integer(row, 'control_activity_recovery_failures'), + 0 + ), + databasePoolWaiting: Math.max( + 0, + ...valid.map((row) => integer(row, 'database_pool_waiting')) + ), + databasePoolWaitersMax: Math.max( + 0, + ...valid.map((row) => integer(row, 'database_pool_waiters_max')) + ), + databasePoolWaitMsMax: Math.max( + 0, + ...valid.map((row) => integer(row, 'database_pool_wait_ms_max')) + ) + } + } + + async claimRegionalRehome( + processSafety?: RegionalRehomeSafetySnapshot + ): Promise { + const now = this.now() + // Directors poll every second; avoid taking the global worker-row lock while disabled. + const control = ( + await this.database.query( + `SELECT enabled, not_before + FROM relay_region_rehome_control + WHERE control_id = 'global'` + ) + )[0] + if (!control) { + await this.initializeRegionalRehomeControl(this.database, now) + return null + } + if (integer(control, 'enabled') !== 1 || integer(control, 'not_before') > now) { + return null + } + this.pendingRegionalRehomeDisableLog = null + const candidateSkips: RegionalRehomeCandidateSkip[] = [] + const claimResult = await this.database.transaction(async (transaction) => { + candidateSkips.length = 0 + await this.initializeRegionalRehomeControl(transaction, now) + const control = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'` + ) + )[0]! + if (integer(control, 'enabled') !== 1 || integer(control, 'not_before') > now) { + return null + } + const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute')) + const preferenceCutoff = now - integer(control, 'preference_max_age_ms') + await transaction.query( + `INSERT INTO relay_region_rehome_worker_state + (worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at) + VALUES ('global', 0, 0, 0, ?) + ON CONFLICT (worker_id) DO NOTHING`, + [now] + ) + const worker = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` + ) + )[0]! + if ( + integer(worker, 'paused_until') > now || + integer(worker, 'next_dispatch_at') > now + ) { + return null + } + const effectiveProcessSafety = processSafety ?? cleanRegionalRehomeSafety(now) + const fleetSafety = await this.readRegionalRehomeFleetSafety(transaction, now) + if ( + !(await this.regionalRehomeSafetyAllowsClaim( + transaction, + worker, + effectiveProcessSafety, + fleetSafety, + now + )) + ) { + return null + } + const retry = ( + await transaction.queryLocked( + `SELECT attempt.*, source.cell_url AS source_cell_url + FROM relay_region_rehome_attempts attempt + JOIN relay_cells source ON source.cell_id = attempt.source_cell_id + JOIN relay_cell_runtime runtime ON runtime.cell_id = attempt.source_cell_id + JOIN relay_cell_capabilities capability + ON capability.cell_id = runtime.cell_id + AND capability.cell_incarnation = runtime.cell_incarnation + JOIN relay_assignment_migrations migration + ON migration.user_id = attempt.user_id + AND migration.relay_host_id = attempt.relay_host_id + AND migration.assignment_epoch = attempt.assignment_epoch + WHERE attempt.drain_receipt_at IS NULL + AND attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + AND attempt.send_attempts < 10 + AND (attempt.last_send_attempt_at IS NULL OR attempt.last_send_attempt_at <= ?) + AND runtime.cell_incarnation = attempt.source_cell_incarnation + AND runtime.ready = 1 AND runtime.last_heartbeat_at > ? + AND capability.regional_rehome_protocol >= 1 + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + ORDER BY attempt.created_at, attempt.attempt_id + LIMIT 1`, + [now - 30_000, now - this.heartbeatTtlMs] + ) + )[0] + if (retry) { + const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now) + if ( + !(await this.regionalRehomeSafetyAllowsClaim( + transaction, + worker, + effectiveProcessSafety, + fleetSafety, + now + )) + ) { + return null + } + await this.markRegionalRehomeDispatchClaimed( + transaction, + text(retry, 'attempt_id'), + now, + intervalMs + ) + retry.send_attempts = integer(retry, 'send_attempts') + 1 + return regionalRehomeAttempt(retry) + } + + // A drain receipt is not convergence: grace enforcement lives only in + // source-cell session state, and attempts have been observed stalled + // dual-homed well past grace with source leases still renewing. Such + // attempts are re-dispatched with the remaining (zero) grace so the + // source force-closes and the host re-resolves onto its registered + // target. + const redrain = ( + await transaction.queryLocked( + `SELECT attempt.*, source.cell_url AS source_cell_url + FROM relay_region_rehome_attempts attempt + JOIN relay_cells source ON source.cell_id = attempt.source_cell_id + JOIN relay_cell_runtime runtime ON runtime.cell_id = attempt.source_cell_id + JOIN relay_cell_capabilities capability + ON capability.cell_id = runtime.cell_id + AND capability.cell_incarnation = runtime.cell_incarnation + JOIN relay_assignment_migrations migration + ON migration.user_id = attempt.user_id + AND migration.relay_host_id = attempt.relay_host_id + AND migration.assignment_epoch = attempt.assignment_epoch + WHERE attempt.drain_receipt_at IS NOT NULL + AND attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + AND attempt.created_at + attempt.drain_grace_ms <= ? + AND attempt.send_attempts < ? + AND (attempt.last_send_attempt_at IS NULL OR attempt.last_send_attempt_at <= ?) + AND runtime.cell_incarnation = attempt.source_cell_incarnation + AND runtime.ready = 1 AND runtime.last_heartbeat_at > ? + AND capability.regional_rehome_protocol >= 1 + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + AND migration.target_registered_at IS NOT NULL + AND EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases source_lease + WHERE source_lease.user_id = attempt.user_id + AND source_lease.relay_host_id = attempt.relay_host_id + AND source_lease.cell_id = attempt.source_cell_id + ) + ORDER BY attempt.created_at, attempt.attempt_id + LIMIT 1`, + [ + now, + REGIONAL_REHOME_REDRAIN_SEND_LIMIT, + now - REGIONAL_REHOME_REDRAIN_INTERVAL_MS, + now - this.heartbeatTtlMs + ] + ) + )[0] + if (redrain) { + const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now) + if ( + !(await this.regionalRehomeSafetyAllowsClaim( + transaction, + worker, + effectiveProcessSafety, + fleetSafety, + now + )) + ) { + return null + } + await this.markRegionalRehomeDispatchClaimed( + transaction, + text(redrain, 'attempt_id'), + now, + intervalMs + ) + redrain.send_attempts = integer(redrain, 'send_attempts') + 1 + redrain.drain_grace_ms = 0 + return regionalRehomeAttempt(redrain) + } + + const candidates = await transaction.query( + `SELECT preference.user_id, preference.relay_host_id, + preference.observed_at, assignment.cell_id AS source_cell_id, + assignment.assignment_epoch + FROM relay_assignment_region_preferences preference + JOIN relay_assignments assignment + ON assignment.user_id = preference.user_id + AND assignment.relay_host_id = preference.relay_host_id + JOIN relay_cell_regions region ON region.cell_id = assignment.cell_id + JOIN relay_cell_admission admission ON admission.cell_id = assignment.cell_id + JOIN relay_cell_runtime runtime ON runtime.cell_id = assignment.cell_id + JOIN relay_cell_capabilities capability + ON capability.cell_id = runtime.cell_id + AND capability.cell_incarnation = runtime.cell_incarnation + WHERE preference.preferred_region = 'asia-east2' + AND preference.observed_at >= ? + AND region.region = 'us-central1' + AND admission.admission_state = 'general' + AND runtime.ready = 1 AND runtime.last_heartbeat_at > ? + AND capability.regional_rehome_protocol >= 1 + AND EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases control + WHERE control.user_id = assignment.user_id + AND control.relay_host_id = assignment.relay_host_id + AND control.cell_id = assignment.cell_id + AND control.activity_kind = 'control' + AND control.activity_id NOT LIKE 'control-pending:%' + AND control.expires_at > ? + AND control.updated_at >= runtime.started_at + ) + AND NOT EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + ) + ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id + LIMIT 10`, + [preferenceCutoff, now - this.heartbeatTtlMs, now] + ) + for (const candidate of candidates) { + const claimed = await this.startRegionalRehomeCandidate(transaction, { + identity: { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + }, + sourceCellId: text(candidate, 'source_cell_id'), + assignmentEpoch: integer(candidate, 'assignment_epoch'), + preferenceCutoff, + drainGraceMs: integer(control, 'drain_grace_ms'), + processSafety: effectiveProcessSafety, + worker, + now, + skips: candidateSkips + }) + if (!claimed) continue + await this.markRegionalRehomeDispatchClaimed( + transaction, + claimed.attemptId, + now, + intervalMs + ) + return { ...claimed, sendAttempts: 1 } + } + if (candidates.length > 0) { + // Skipped candidates still cost all-rows FOR UPDATE inventory scans; + // charge the dispatch interval so skips are rate-limited like claims. + await this.markRegionalRehomeTickSkipped(transaction, now, intervalMs) + } + return null + }) + const pendingDisableLog = this.pendingRegionalRehomeDisableLog + this.pendingRegionalRehomeDisableLog = null + if (pendingDisableLog) console.warn(JSON.stringify(pendingDisableLog)) + if (claimResult === null && candidateSkips.length > 0) { + console.warn(JSON.stringify(aggregateRegionalRehomeCandidateSkips(candidateSkips))) + } + return claimResult + } + + private async startRegionalRehomeCandidate( + transaction: RelayDatabase, + input: { + identity: AssignmentIdentity + sourceCellId: string + assignmentEpoch: number + preferenceCutoff: number + drainGraceMs: number + processSafety: RegionalRehomeSafetySnapshot + worker: SqlRow + now: number + skips: RegionalRehomeCandidateSkip[] + } + ): Promise | null> { + const assignment = await this.assignmentRow(transaction, input.identity) + if ( + !assignment || + text(assignment, 'cell_id') !== input.sourceCellId || + integer(assignment, 'assignment_epoch') !== input.assignmentEpoch + ) { + input.skips.push({ reason: 'candidate_stale' }) + return null + } + const preference = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_region_preferences + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + )[0] + if ( + !preference || + text(preference, 'preferred_region') !== 'asia-east2' || + integer(preference, 'observed_at') < input.preferenceCutoff + ) { + input.skips.push({ reason: 'candidate_stale' }) + return null + } + const activeMigration = await transaction.queryLocked( + `SELECT assignment_epoch FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? + AND completed_at IS NULL AND aborted_at IS NULL`, + [input.identity.userId, input.identity.relayHostId] + ) + if (activeMigration.length > 0) { + input.skips.push({ reason: 'candidate_stale' }) + return null + } + const activityLeases = await this.lockAssignmentActivities(transaction, input.identity) + assertAssignmentActivityCounts(assignment, activityLeases, 0) + const cells = await this.lockCellInventory(transaction) + const admission = await cellAdmissionStates(transaction) + const regions = new Map( + (await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [ + text(row, 'cell_id'), + relayRegion(row, 'region') + ]) + ) + const runtimes = await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime ORDER BY cell_id` + ) + const capabilities = await transaction.queryLocked( + `SELECT * FROM relay_cell_capabilities ORDER BY cell_id` + ) + const safetyRows = await transaction.queryLocked( + `SELECT * FROM relay_cell_rehome_safety ORDER BY cell_id` + ) + const source = cells.find((row) => text(row, 'cell_id') === input.sourceCellId) + const sourceRuntime = runtimes.find( + (row) => text(row, 'cell_id') === input.sourceCellId + ) + const sourceCapability = capabilities.find( + (row) => text(row, 'cell_id') === input.sourceCellId + ) + const sourceSafety = safetyRows.find( + (row) => text(row, 'cell_id') === input.sourceCellId + ) + const fleetSafety = regionalRehomeFleetSafetyFromInventory({ + cells, + admission, + regions, + runtimes, + capabilities, + safetyRows, + now: input.now, + heartbeatTtlMs: this.heartbeatTtlMs + }) + const safetyFailure = regionalRehomeFleetSafetyFailure( + input.processSafety, + fleetSafety, + input.now + ) + if (safetyFailure) { + await this.pauseRegionalRehomeForSafety( + transaction, + input.worker, + input.now, + safetyFailure, + fleetSafety + ) + return null + } + if ( + !source || + integer(source, 'enabled') !== 1 || + admission.get(input.sourceCellId) !== 'general' || + regions.get(input.sourceCellId) !== RELAY_DEFAULT_REGION || + !sourceRuntime || + integer(sourceRuntime, 'ready') !== 1 || + integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs || + !sourceCapability || + text(sourceCapability, 'cell_incarnation') !== + text(sourceRuntime, 'cell_incarnation') || + integer(sourceCapability, 'regional_rehome_protocol') < 1 + ) { + input.skips.push({ reason: 'source_ineligible', cellId: input.sourceCellId }) + return null + } + if (!regionalRehomeCellSafetyIsClean(sourceSafety, sourceRuntime, input.now)) { + input.skips.push(cellUncleanSkip('source_unclean', input.sourceCellId, sourceSafety)) + return null + } + const sourceControlActive = activityLeases.some( + (lease) => + text(lease, 'cell_id') === input.sourceCellId && + text(lease, 'activity_kind') === 'control' && + !text(lease, 'activity_id').startsWith('control-pending:') && + integer(lease, 'expires_at') > input.now && + integer(lease, 'updated_at') >= integer(sourceRuntime, 'started_at') + ) + if (!sourceControlActive) { + input.skips.push({ reason: 'source_control_inactive', cellId: input.sourceCellId }) + return null + } + const connectionHeadroom = await this.connectionHeadroomByCell(transaction) + const eligibleTargets = cells.filter((row) => { + const cellId = text(row, 'cell_id') + const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId) + return ( + cellId !== input.sourceCellId && + integer(row, 'enabled') === 1 && + admission.get(cellId) === 'general' && + regions.get(cellId) === 'asia-east2' && + runtime !== undefined && + integer(runtime, 'ready') === 1 && + integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs + ) + }) + const targetIsClean = (row: SqlRow): boolean => { + const cellId = text(row, 'cell_id') + return regionalRehomeCellSafetyIsClean( + safetyRows.find((safety) => text(safety, 'cell_id') === cellId), + runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)!, + input.now + ) + } + const targetCandidates = eligibleTargets.filter( + (row) => targetIsClean(row) && connectionHeadroom.get(text(row, 'cell_id')) !== false + ) + if (targetCandidates.length === 0) { + const unclean = eligibleTargets.filter((row) => !targetIsClean(row)) + for (const row of unclean) { + const cellId = text(row, 'cell_id') + input.skips.push( + cellUncleanSkip( + 'target_unclean', + cellId, + safetyRows.find((safety) => text(safety, 'cell_id') === cellId) + ) + ) + } + if (unclean.length === 0) { + input.skips.push({ + reason: eligibleTargets.length === 0 ? 'no_eligible_target' : 'no_target_headroom' + }) + } + return null + } + targetCandidates.sort((left, right) => { + const leftRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === text(left, 'cell_id') + )! + const rightRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === text(right, 'cell_id') + )! + const leftLoad = + (integer(left, 'reserved_requests') + integer(leftRuntime, 'observed_requests')) / + integer(left, 'capacity_requests') + const rightLoad = + (integer(right, 'reserved_requests') + integer(rightRuntime, 'observed_requests')) / + integer(right, 'capacity_requests') + return leftLoad - rightLoad || + text(left, 'cell_id').localeCompare(text(right, 'cell_id')) + }) + const sourceRequestUnits = activityUnitsForCell(activityLeases, input.sourceCellId) + const targetReservedUnits = sourceRequestUnits + 1 + const target = targetCandidates.find( + (row) => + integer(row, 'reserved_requests') + targetReservedUnits <= + integer(row, 'capacity_requests') + ) + if (!target) { + input.skips.push({ reason: 'no_target_headroom' }) + return null + } + const targetCellId = text(target, 'cell_id') + const targetRuntime = runtimes.find( + (runtime) => text(runtime, 'cell_id') === targetCellId + )! + await this.adjustCellReservation(transaction, targetCellId, targetReservedUnits) + const previousEpoch = integer(assignment, 'assignment_epoch') + const assignmentEpoch = previousEpoch + 1 + const expiresAt = input.now + ASSIGNMENT_LIMITS.migrationLeaseMs + await transaction.query( + `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, + reserved_controls = reserved_controls + 1, + migration_leases = migration_leases + 1, + lease_expires_at = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + targetCellId, + assignmentEpoch, + expiresAt, + input.now, + input.identity.userId, + input.identity.relayHostId + ] + ) + await transaction.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, 'control', ?, 1, ?, ?), + (?, ?, ?, 'migration', ?, ?, ?, ?)`, + [ + input.identity.userId, + input.identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + targetCellId, + expiresAt, + input.now, + input.identity.userId, + input.identity.relayHostId, + migrationActivityId(assignmentEpoch), + targetCellId, + sourceRequestUnits, + expiresAt, + input.now + ] + ) + await this.insertControlConnectionReservation( + transaction, + input.identity, + targetCellId, + assignmentEpoch, + expiresAt, + input.now + ) + await transaction.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`, + [ + input.identity.userId, + input.identity.relayHostId, + input.sourceCellId, + targetCellId, + previousEpoch, + assignmentEpoch, + sourceRequestUnits, + targetReservedUnits, + expiresAt, + input.now, + input.now + ] + ) + await transaction.query( + `INSERT INTO relay_assignment_migration_incarnations + (user_id, relay_host_id, assignment_epoch, source_cell_incarnation, + target_cell_incarnation) + VALUES (?, ?, ?, ?, ?)`, + [ + input.identity.userId, + input.identity.relayHostId, + assignmentEpoch, + text(sourceRuntime, 'cell_incarnation'), + text(targetRuntime, 'cell_incarnation') + ] + ) + const attemptId = randomUUID() + await transaction.query( + `INSERT INTO relay_region_rehome_attempts + (attempt_id, user_id, relay_host_id, preferred_region, + source_cell_id, source_cell_incarnation, target_cell_id, + target_cell_incarnation, previous_epoch, assignment_epoch, + drain_grace_ms, send_attempts, last_send_attempt_at, + drain_receipt_at, drain_outcome, completed_at, aborted_at, + created_at, updated_at) + VALUES (?, ?, ?, 'asia-east2', ?, ?, ?, ?, ?, ?, ?, 0, NULL, + NULL, NULL, NULL, NULL, ?, ?)`, + [ + attemptId, + input.identity.userId, + input.identity.relayHostId, + input.sourceCellId, + text(sourceRuntime, 'cell_incarnation'), + targetCellId, + text(targetRuntime, 'cell_incarnation'), + previousEpoch, + assignmentEpoch, + input.drainGraceMs, + input.now, + input.now + ] + ) + return { + ...input.identity, + attemptId, + preferredRegion: 'asia-east2', + sourceCellId: input.sourceCellId, + sourceCellUrl: text(source, 'cell_url'), + sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'), + targetCellId, + targetCellIncarnation: text(targetRuntime, 'cell_incarnation'), + previousEpoch, + assignmentEpoch, + drainGraceMs: input.drainGraceMs + } + } + + private async lockedRegionalRehomeFleetSafety( + transaction: RelayDatabase, + now: number + ): Promise { + const cells = await this.lockCellInventory(transaction) + const admission = await cellAdmissionStates(transaction) + const regions = new Map( + (await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [ + text(row, 'cell_id'), + relayRegion(row, 'region') + ]) + ) + const runtimes = await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime ORDER BY cell_id` + ) + const capabilities = await transaction.queryLocked( + `SELECT * FROM relay_cell_capabilities ORDER BY cell_id` + ) + const safetyRows = await transaction.queryLocked( + `SELECT * FROM relay_cell_rehome_safety ORDER BY cell_id` + ) + return regionalRehomeFleetSafetyFromInventory({ + cells, + admission, + regions, + runtimes, + capabilities, + safetyRows, + now, + heartbeatTtlMs: this.heartbeatTtlMs + }) + } + + private async regionalRehomeSafetyAllowsClaim( + transaction: RelayDatabase, + worker: SqlRow, + processSafety: RegionalRehomeSafetySnapshot, + fleetSafety: RegionalRehomeFleetSafety, + now: number + ): Promise { + const failure = regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now) + if (!failure) { + return true + } + await this.pauseRegionalRehomeForSafety(transaction, worker, now, failure, fleetSafety) + return false + } + + private async pauseRegionalRehomeForSafety( + transaction: RelayDatabase, + worker: SqlRow, + now: number, + reason: string, + fleetSafety: RegionalRehomeFleetSafety + ): Promise { + const disabled = await transaction.query( + `UPDATE relay_region_rehome_control + SET generation = generation + 1, enabled = 0, updated_at = ? + WHERE control_id = 'global' AND enabled = 1 + RETURNING generation`, + [now] + ) + // The durable disable is otherwise invisible: nothing else records why + // claims stopped and inspection only shows enabled=false. Logged after + // the transaction commits so a rollback cannot fabricate the record. + if (disabled.length > 0) { + this.pendingRegionalRehomeDisableLog = { + event: 'orca_relay_regional_rehome_safety_disabled', + reason, + controlGeneration: integer(disabled[0]!, 'generation'), + now, + requiredCells: fleetSafety.requiredCells, + missingCells: fleetSafety.missingCells, + observedAt: fleetSafety.observedAt, + sqlFailures: fleetSafety.sqlFailures, + reconnects: fleetSafety.reconnects, + maxReconnects: fleetSafety.maxReconnects, + controlActivityRecoveryFailures: fleetSafety.controlActivityRecoveryFailures, + databasePoolWaiting: fleetSafety.databasePoolWaiting, + databasePoolWaitersMax: fleetSafety.databasePoolWaitersMax, + databasePoolWaitMsMax: fleetSafety.databasePoolWaitMsMax + } + } + await this.incrementRegionalRehomeWorkerFailure(transaction, worker, now) + } + + private async markRegionalRehomeTickSkipped( + transaction: RelayDatabase, + now: number, + intervalMs: number + ): Promise { + await transaction.query( + `UPDATE relay_region_rehome_worker_state + SET next_dispatch_at = ?, updated_at = ? WHERE worker_id = 'global'`, + [now + intervalMs, now] + ) + } + + private async markRegionalRehomeDispatchClaimed( + transaction: RelayDatabase, + attemptId: string, + now: number, + intervalMs: number + ): Promise { + await transaction.query( + `UPDATE relay_region_rehome_attempts + SET send_attempts = send_attempts + 1, last_send_attempt_at = ?, updated_at = ? + WHERE attempt_id = ?`, + [now, now, attemptId] + ) + await transaction.query( + `UPDATE relay_region_rehome_worker_state + SET next_dispatch_at = ?, updated_at = ? WHERE worker_id = 'global'`, + [now + intervalMs, now] + ) + } + + async recordRegionalRehomeDrainReceipt( + attemptId: string, + outcome: RegionalHostDrainOutcome + ): Promise { + const now = this.now() + return await this.database.transaction(async (transaction) => { + const worker = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` + ) + )[0] + const attempt = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [attemptId] + ) + )[0] + if (!attempt) throw new Error('regional_rehome_attempt_not_found') + // Any receipt proves the source cell answered: reset the failure budget + // even when a redrain repeats the stored outcome; otherwise a + // redrain-dominated stream lets scattered transient failures reach the + // durable three-failure disable. + if (worker) { + await transaction.query( + `UPDATE relay_region_rehome_worker_state + SET consecutive_failures = 0, paused_until = 0, updated_at = ? + WHERE worker_id = 'global'`, + [now] + ) + } + const existingOutcome = optionalText(attempt, 'drain_outcome') + if (existingOutcome === outcome) return false + // Redrains produce one receipt per dispatch; the latest outcome wins. + await transaction.query( + `UPDATE relay_region_rehome_attempts + SET drain_receipt_at = ?, drain_outcome = ?, updated_at = ? + WHERE attempt_id = ?`, + [now, outcome, now, attemptId] + ) + return true + }) + } + + async recordRegionalRehomeDispatchFailure(attemptId: string): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + const worker = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` + ) + )[0] + const attempt = ( + await transaction.queryLocked( + `SELECT attempt_id FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [attemptId] + ) + )[0] + if (!worker || !attempt) return + await this.incrementRegionalRehomeWorkerFailure(transaction, worker, now) + }) + } + + async recordRegionalRehomeWorkerFailure(): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + await transaction.query( + `INSERT INTO relay_region_rehome_worker_state + (worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at) + VALUES ('global', 0, 0, 0, ?) + ON CONFLICT (worker_id) DO NOTHING`, + [now] + ) + const worker = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` + ) + )[0]! + await this.incrementRegionalRehomeWorkerFailure(transaction, worker, now) + }) + } + + private async incrementRegionalRehomeWorkerFailure( + transaction: RelayDatabase, + worker: SqlRow, + now: number + ): Promise { + const failures = integer(worker, 'consecutive_failures') + 1 + await transaction.query( + `UPDATE relay_region_rehome_worker_state + SET consecutive_failures = ?, paused_until = ?, updated_at = ? + WHERE worker_id = 'global'`, + [failures, failures >= 3 ? now + 5 * 60_000 : 0, now] + ) + if (failures >= 3) { + await transaction.query( + `UPDATE relay_region_rehome_control + SET generation = generation + 1, enabled = 0, updated_at = ? + WHERE control_id = 'global' AND enabled = 1`, + [now] + ) + } + } + + async completeReadyRegionalRehomes(limit = 10): Promise { + const now = this.now() + const quarantined = this.quarantinedRegionalRehomeAttemptIds(now) + const exclusion = quarantined.length + ? ` AND attempt.attempt_id NOT IN (${quarantined.map(() => '?').join(', ')})` + : '' + const candidates = await this.database.query( + `SELECT attempt.attempt_id, attempt.user_id, attempt.relay_host_id, + attempt.assignment_epoch + FROM relay_region_rehome_attempts attempt + JOIN relay_assignment_migrations migration + ON migration.user_id = attempt.user_id + AND migration.relay_host_id = attempt.relay_host_id + AND migration.assignment_epoch = attempt.assignment_epoch + WHERE attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + AND migration.target_registered_at IS NOT NULL + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases source_lease + WHERE source_lease.user_id = attempt.user_id + AND source_lease.relay_host_id = attempt.relay_host_id + AND source_lease.cell_id = attempt.source_cell_id + )${exclusion} + ORDER BY attempt.created_at, attempt.attempt_id + LIMIT ?`, + [...quarantined, limit] + ) + let completed = 0 + for (const candidate of candidates) { + // One poisoned row must not stall every later candidate: an invariant + // throw here blocked fleet completions head-of-line in production. + const attemptId = text(candidate, 'attempt_id') + try { + const changed = await this.completeRegionalRehomeCandidate( + { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + }, + integer(candidate, 'assignment_epoch'), + now + ) + if (changed) completed++ + this.regionalRehomeCandidateQuarantine.delete(attemptId) + } catch (error) { + this.recordRegionalRehomeCandidateFailure('complete', attemptId, now, error) + } + } + return completed + } + + // Repeated invariant failures quarantine the attempt out of the sweeps' + // LIMIT pages: poisoned rows are permanent and always the oldest, so + // without exclusion they eventually starve every healthy candidate. + private recordRegionalRehomeCandidateFailure( + operation: 'complete' | 'abort', + attemptId: string, + now: number, + error: unknown + ): void { + const entry = this.regionalRehomeCandidateQuarantine.get(attemptId) ?? { + failures: 0, + until: 0 + } + entry.failures++ + if (entry.failures >= REGIONAL_REHOME_QUARANTINE_FAILURES) { + entry.until = now + REGIONAL_REHOME_QUARANTINE_MS + } + this.regionalRehomeCandidateQuarantine.delete(attemptId) + this.regionalRehomeCandidateQuarantine.set(attemptId, entry) + if (this.regionalRehomeCandidateQuarantine.size > REGIONAL_REHOME_QUARANTINE_MEMORY_LIMIT) { + // Evict a non-quarantined entry first: mid-quarantine rows are excluded + // from the candidate pages and losing one returns it to the page with a + // reset counter. + let evict = this.regionalRehomeCandidateQuarantine.keys().next().value! + for (const [key, candidate] of this.regionalRehomeCandidateQuarantine) { + if (candidate.until <= now) { + evict = key + break + } + } + this.regionalRehomeCandidateQuarantine.delete(evict) + } + warnRegionalRehomeCandidateFailure(operation, attemptId, error) + } + + private quarantinedRegionalRehomeAttemptIds(now: number): string[] { + const excluded: string[] = [] + for (const [attemptId, entry] of this.regionalRehomeCandidateQuarantine) { + if (entry.until > now) excluded.push(attemptId) + if (excluded.length >= REGIONAL_REHOME_QUARANTINE_EXCLUSION_LIMIT) break + } + return excluded + } + + async refreshRegionalRehomeLeases(limit = 100): Promise { + const now = this.now() + const candidates = await this.database.query( + `SELECT user_id, relay_host_id, assignment_epoch + FROM relay_region_rehome_attempts + WHERE completed_at IS NULL AND aborted_at IS NULL + ORDER BY created_at, attempt_id LIMIT ?`, + [limit] + ) + let refreshed = 0 + for (const candidate of candidates) { + const identity = { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + } + const assignmentEpoch = integer(candidate, 'assignment_epoch') + const changed = await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + const attempt = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_attempts + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + const migration = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + if ( + !assignment || + !attempt || + !migration || + optionalInteger(attempt, 'completed_at') !== undefined || + optionalInteger(attempt, 'aborted_at') !== undefined || + optionalInteger(migration, 'completed_at') !== undefined || + optionalInteger(migration, 'aborted_at') !== undefined + ) { + return false + } + const attemptAgeMs = now - integer(attempt, 'created_at') + if (attemptAgeMs >= REGIONAL_REHOME_MAX_REFRESH_MS) { + return false + } + if ( + optionalInteger(migration, 'target_registered_at') === undefined && + attemptAgeMs >= REGIONAL_REHOME_UNREGISTERED_REFRESH_MS + ) { + await transaction.query( + `UPDATE relay_assignment_activity_leases + SET expires_at = CASE WHEN expires_at < ? THEN expires_at ELSE ? END, + updated_at = ? + WHERE user_id = ? AND relay_host_id = ? + AND activity_id IN (?, ?)`, + [ + now, + now, + now, + identity.userId, + identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ] + ) + await transaction.query( + `UPDATE relay_assignment_migrations + SET expires_at = CASE WHEN expires_at < ? THEN expires_at ELSE ? END, + updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return false + } + const leases = await this.lockAssignmentActivities(transaction, identity) + const protectedIds = new Set([ + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ]) + const protectedLeases = leases.filter((lease) => + protectedIds.has(text(lease, 'activity_id')) + ) + if (protectedLeases.length === 0) return false + const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs + await transaction.query( + `UPDATE relay_assignment_activity_leases + SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? + AND activity_id IN (?, ?)`, + [ + expiresAt, + now, + identity.userId, + identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ] + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + await transaction.query( + `UPDATE relay_assignments SET lease_expires_at = + CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, + last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [expiresAt, expiresAt, now, identity.userId, identity.relayHostId] + ) + return true + }) + if (changed) refreshed++ + } + return refreshed + } + + private async completeRegionalRehomeCandidate( + identity: AssignmentIdentity, + assignmentEpoch: number, + now: number + ): Promise { + return await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + const attempt = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_attempts + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + const migration = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + if ( + !attempt || + !migration || + optionalInteger(attempt, 'completed_at') !== undefined || + optionalInteger(attempt, 'aborted_at') !== undefined || + optionalInteger(migration, 'target_registered_at') === undefined || + optionalInteger(migration, 'completed_at') !== undefined || + optionalInteger(migration, 'aborted_at') !== undefined + ) { + return false + } + const sourceCellId = text(attempt, 'source_cell_id') + const targetCellId = text(attempt, 'target_cell_id') + if ( + !assignment || + text(assignment, 'cell_id') !== targetCellId || + integer(assignment, 'assignment_epoch') !== assignmentEpoch || + text(migration, 'source_cell_id') !== sourceCellId || + text(migration, 'target_cell_id') !== targetCellId + ) { + throw new Error('regional_rehome_assignment_mismatch') + } + const leases = await this.lockAssignmentActivities(transaction, identity) + if (activityUnitsForCell(leases, sourceCellId) !== 0) return false + await this.repairAssignmentActivityCounts( + transaction, + identity, + text(attempt, 'attempt_id'), + assignment, + leases, + migration + ) + const cells = await this.lockCellInventory(transaction) + const target = cells.find((cell) => text(cell, 'cell_id') === targetCellId) + const admission = await cellAdmissionStates(transaction) + if ( + !target || + integer(target, 'enabled') !== 1 || + !['general', 'migration-only'].includes(admission.get(targetCellId) ?? '') + ) { + return false + } + const targetRuntime = ( + await transaction.queryLocked( + `SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, + [targetCellId] + ) + )[0] + if ( + !targetRuntime || + integer(targetRuntime, 'ready') !== 1 || + integer(targetRuntime, 'last_heartbeat_at') <= now - this.heartbeatTtlMs + ) { + return false + } + const targetActive = leases.some( + (lease) => + text(lease, 'cell_id') === targetCellId && + text(lease, 'activity_kind') === 'control' && + !text(lease, 'activity_id').startsWith('control-pending:') && + integer(lease, 'expires_at') > now && + integer(lease, 'updated_at') >= integer(targetRuntime, 'started_at') + ) + if (!targetActive) return false + const migrationLease = activityLeaseById( + leases, + migrationActivityId(assignmentEpoch) + ) + if (migrationLease) { + await this.removeActivityLease(transaction, identity, migrationLease, now) + } + await transaction.query( + `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + await transaction.query( + `UPDATE relay_region_rehome_attempts SET completed_at = ?, updated_at = ? + WHERE attempt_id = ?`, + [now, now, text(attempt, 'attempt_id')] + ) + return true + }) + } + + // Why: counter skew left by a pre-fix sticky grant is reconstructible from the + // locked lease rows; lease shape and migration topology are not, so only a + // count mismatch is repaired and the re-assert still throws on anything else. + // reconcileReservationAccounting cannot stand in: it opens its own + // transaction, while this has to run inside the sweep's on rows it holds. + private async repairAssignmentActivityCounts( + transaction: RelayDatabase, + identity: AssignmentIdentity, + attemptId: string, + assignment: SqlRow, + leases: SqlRow[], + migrationRow: SqlRow + ): Promise { + try { + assertAssignmentActivityAccounting(assignment, leases, migrationRow) + return + } catch (error) { + if ( + !(error instanceof Error) || + error.message !== 'migration_activity_accounting_mismatch' + ) { + throw error + } + } + const counts = activityCounts(leases) + await transaction.query( + `UPDATE relay_assignments SET reserved_controls = ?, reserved_splices = ?, + reserved_invites = ?, pending_installs = ?, pending_confirmations = ?, + migration_leases = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + counts.control, + counts.splice, + counts.invite, + counts.install, + counts.confirmation, + counts.migration, + identity.userId, + identity.relayHostId + ] + ) + const repaired = await this.assignmentRow(transaction, identity) + if (!repaired) throw new Error('regional_rehome_assignment_mismatch') + assertAssignmentActivityAccounting(repaired, leases, migrationRow) + noteRegionalRehomeActivityCountsRepaired(attemptId) + } + + async reapRegionalRehomeAttempts(): Promise { + const now = this.now() + const completed = await this.database.query( + `UPDATE relay_region_rehome_attempts + SET completed_at = COALESCE(completed_at, ?), updated_at = ? + WHERE completed_at IS NULL AND aborted_at IS NULL + AND EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = relay_region_rehome_attempts.user_id + AND migration.relay_host_id = relay_region_rehome_attempts.relay_host_id + AND migration.assignment_epoch = relay_region_rehome_attempts.assignment_epoch + AND migration.completed_at IS NOT NULL + )`, + [now, now] + ) + const aborted = await this.database.query( + `UPDATE relay_region_rehome_attempts + SET aborted_at = COALESCE(aborted_at, ?), updated_at = ? + WHERE completed_at IS NULL AND aborted_at IS NULL + AND EXISTS ( + SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = relay_region_rehome_attempts.user_id + AND migration.relay_host_id = relay_region_rehome_attempts.relay_host_id + AND migration.assignment_epoch = relay_region_rehome_attempts.assignment_epoch + AND migration.aborted_at IS NOT NULL + )`, + [now, now] + ) + if (integer(aborted[0]!, 'changes') > 0) { + await this.disableRegionalRehomeControl() + } + return integer(completed[0]!, 'changes') + integer(aborted[0]!, 'changes') + } + + async abortExpiredRegionalRehomes(limit = 100): Promise { + const now = this.now() + const quarantined = this.quarantinedRegionalRehomeAttemptIds(now) + const exclusion = quarantined.length + ? ` AND attempt_id NOT IN (${quarantined.map(() => '?').join(', ')})` + : '' + const candidates = await this.database.query( + `SELECT attempt_id, user_id, relay_host_id, assignment_epoch + FROM relay_region_rehome_attempts + WHERE completed_at IS NULL AND aborted_at IS NULL + AND created_at <= ?${exclusion} + ORDER BY created_at, attempt_id LIMIT ?`, + [now - REGIONAL_REHOME_MAX_REFRESH_MS, ...quarantined, limit] + ) + let aborted = 0 + for (const candidate of candidates) { + const identity = { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + } + const assignmentEpoch = integer(candidate, 'assignment_epoch') + const attemptId = text(candidate, 'attempt_id') + // Isolated like the completion sweep: one poisoned row must not stall + // every later candidate. + let changed = false + try { + changed = await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + const attempt = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_attempts + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + const migration = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + if ( + !assignment || + !attempt || + !migration || + optionalInteger(attempt, 'completed_at') !== undefined || + optionalInteger(attempt, 'aborted_at') !== undefined || + integer(attempt, 'created_at') > now - REGIONAL_REHOME_MAX_REFRESH_MS || + optionalInteger(migration, 'completed_at') !== undefined || + optionalInteger(migration, 'aborted_at') !== undefined + ) { + return false + } + const sourceCellId = text(attempt, 'source_cell_id') + const targetCellId = text(attempt, 'target_cell_id') + if ( + text(assignment, 'cell_id') !== targetCellId || + integer(assignment, 'assignment_epoch') !== assignmentEpoch + ) { + throw new Error('regional_rehome_assignment_mismatch') + } + const leases = await this.lockAssignmentActivities(transaction, identity) + if (activityUnitsForCell(leases, sourceCellId) > 0) return false + const targetActive = leases.some( + (lease) => + text(lease, 'cell_id') === targetCellId && + text(lease, 'activity_kind') === 'control' && + !text(lease, 'activity_id').startsWith('control-pending:') && + integer(lease, 'expires_at') > now + ) + if (targetActive) return false + const cells = await this.lockCellInventory(transaction) + const source = cells.find((cell) => text(cell, 'cell_id') === sourceCellId) + const admission = await cellAdmissionStates(transaction) + if ( + !source || + integer(source, 'enabled') !== 1 || + admission.get(sourceCellId) !== 'general' || + !(await this.cellIsLive(transaction, sourceCellId, now)) + ) { + return false + } + for (const activityId of [ + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ]) { + const lease = activityLeaseById(leases, activityId) + if (lease) await this.removeActivityLease(transaction, identity, lease, now) + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + targetCellId, + assignmentEpoch, + now + ) + await transaction.query( + `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, + lease_expires_at = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + sourceCellId, + assignmentEpoch + 1, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now, + identity.userId, + identity.relayHostId + ] + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + await transaction.query( + `UPDATE relay_region_rehome_attempts SET aborted_at = ?, updated_at = ? + WHERE attempt_id = ?`, + [now, now, text(attempt, 'attempt_id')] + ) + await transaction.query( + `UPDATE relay_region_rehome_control + SET generation = generation + 1, enabled = 0, updated_at = ? + WHERE control_id = 'global' AND enabled = 1`, + [now] + ) + return true + }) + this.regionalRehomeCandidateQuarantine.delete(attemptId) + } catch (error) { + this.recordRegionalRehomeCandidateFailure('abort', attemptId, now, error) + } + if (changed) aborted++ + } + return aborted + } + + async abortExpiredEvacuations(): Promise { + const now = this.now() + const abandonedBefore = now - STRANDED_MIGRATION_ABANDON_MS + const candidates = await this.database.query( + `SELECT migration.user_id, migration.relay_host_id, migration.assignment_epoch + FROM relay_assignment_migrations migration + WHERE migration.expires_at <= ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + AND ${ABORTABLE_EXPIRED_MIGRATION} + ORDER BY user_id, relay_host_id, assignment_epoch`, + [now, now, abandonedBefore, abandonedBefore] + ) + let aborted = 0 + for (const candidate of candidates) { + const didAbort = await this.database.transaction(async (transaction) => { + const identity = { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + } + // Migration cleanup follows the same assignment-first order as evacuation. + const assignment = await this.assignmentRow(transaction, identity) + const assignmentEpoch = integer(candidate, 'assignment_epoch') + const regionalAttempt = ( + await transaction.queryLocked( + `SELECT attempt_id FROM relay_region_rehome_attempts + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND completed_at IS NULL AND aborted_at IS NULL`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + const row = ( + await transaction.queryLocked( + `SELECT migration.* FROM relay_assignment_migrations migration + WHERE migration.user_id = ? AND migration.relay_host_id = ? + AND migration.assignment_epoch = ? + AND migration.expires_at <= ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + AND ${ABORTABLE_EXPIRED_MIGRATION}`, + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + now, + now, + abandonedBefore, + abandonedBefore + ] + ) + )[0] + if (!row) return false + const targetCellId = text(row, 'target_cell_id') + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + if (!assignment) throw new Error('migration_assignment_missing') + const currentAssignmentEpoch = integer(assignment, 'assignment_epoch') + const assignmentEpochMatches = + text(assignment, 'cell_id') === targetCellId && + currentAssignmentEpoch === assignmentEpoch + const pendingTargetControl = activityLeaseById( + activityLeases, + pendingControlActivityId(assignmentEpoch) + ) + const targetGrantIsFresh = + assignmentEpochMatches && + pendingTargetControl !== undefined && + text(pendingTargetControl, 'cell_id') === targetCellId && + text(pendingTargetControl, 'activity_kind') === 'control' && + integer(pendingTargetControl, 'expires_at') > now + const targetIsActive = activityLeases.some( + (lease) => + text(lease, 'cell_id') === targetCellId && + text(lease, 'activity_kind') === 'control' && + text(lease, 'activity_id') !== pendingControlActivityId(assignmentEpoch) + ) + if (targetGrantIsFresh) return false + if (targetIsActive && assignmentEpochMatches) { + // A committed target control is stronger evidence than a failed follow-up + // write; repair the marker instead of rolling a live desktop backward. + await transaction.query( + `UPDATE relay_assignment_migrations + SET target_registered_at = COALESCE(target_registered_at, ?), updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return false + } + if (!assignmentEpochMatches) { + if (currentAssignmentEpoch <= assignmentEpoch) { + throw new Error('migration_assignment_mismatch') + } + // A newer assignment is authoritative regardless of where it landed. + // Retire only this obsolete migration; never rewrite the newer epoch. + const obsoleteLeases = [ + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ] + .map((activityId) => activityLeaseById(activityLeases, activityId)) + .filter((lease): lease is SqlRow => lease !== undefined) + if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction) + for (const lease of obsoleteLeases) { + await this.removeActivityLease(transaction, identity, lease, now) + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + targetCellId, + assignmentEpoch, + now + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return true + } + const cells = await this.lockCellInventory(transaction) + const sourceCellId = text(row, 'source_cell_id') + const admissionRows = await transaction.query( + `SELECT cell_id, admission_state, updated_at FROM relay_cell_admission + WHERE cell_id IN (?, ?)`, + [sourceCellId, targetCellId] + ) + const sourceCell = cells.find((cell) => text(cell, 'cell_id') === sourceCellId) + const targetCell = cells.find((cell) => text(cell, 'cell_id') === targetCellId) + const sourceAdmission = admissionRows.find( + (admission) => text(admission, 'cell_id') === sourceCellId + ) + const targetAdmission = admissionRows.find( + (admission) => text(admission, 'cell_id') === targetCellId + ) + const registered = optionalInteger(row, 'target_registered_at') !== undefined + const sourceIsDurablyFenced = + registered && + ( + await transaction.query( + `SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = ? AND migration.relay_host_id = ? + AND migration.assignment_epoch = ? + AND ${DURABLY_FENCED_MIGRATION_SOURCE}`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + ).length === 1 + const retireOnTarget = + registered && + activityUnitsForCell(activityLeases, sourceCellId) === 0 && + sourceCell !== undefined && + integer(sourceCell, 'enabled') === 0 && + sourceAdmission !== undefined && + text(sourceAdmission, 'admission_state') === 'existing-only' && + (integer(sourceAdmission, 'updated_at') <= abandonedBefore || + sourceIsDurablyFenced) && + targetCell !== undefined && + integer(targetCell, 'enabled') === 1 && + targetAdmission !== undefined && + ['migration-only', 'general'].includes(text(targetAdmission, 'admission_state')) + const rollbackReason = + !registered || + (targetCell !== undefined && + integer(targetCell, 'enabled') === 0 && + targetAdmission !== undefined && + text(targetAdmission, 'admission_state') === 'existing-only' && + integer(targetAdmission, 'updated_at') <= abandonedBefore) + const regionalRollbackSourceAvailable = + !regionalAttempt || + (sourceCell !== undefined && + integer(sourceCell, 'enabled') === 1 && + sourceAdmission !== undefined && + text(sourceAdmission, 'admission_state') === 'general' && + (await this.cellIsLive(transaction, sourceCellId, now))) + const rollbackToSource = rollbackReason && regionalRollbackSourceAvailable + if (!retireOnTarget && !rollbackToSource) return false + for (const activityId of [ + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ]) { + const lease = activityLeaseById(activityLeases, activityId) + if (lease) await this.removeActivityLease(transaction, identity, lease, now) + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + targetCellId, + assignmentEpoch, + now + ) + if (retireOnTarget) { + await transaction.query( + `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return true + } + await transaction.query( + `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, + lease_expires_at = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + sourceCellId, + assignmentEpoch + 1, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now, + identity.userId, + identity.relayHostId + ] + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return true + }) + if (didAbort) aborted++ + } + return aborted + } + + async releaseExpiredActivityLeases(): Promise { + const now = this.now() + await this.database.query( + `UPDATE relay_control_connection_reservations + SET state = 'late-arrival-debt', updated_at = ? + WHERE state = 'reserved' AND timeout_at <= ?`, + [now, now] + ) + await this.database.query( + `UPDATE relay_control_connection_reservations + SET state = 'released', released_at = ?, updated_at = ? + WHERE state = 'late-arrival-debt' + AND claim_activity_id IS NULL + AND timeout_at <= ?`, + [now, now, now - LATE_ARRIVAL_DEBT_RETENTION_MS] + ) + const candidates = await this.database.query( + `SELECT user_id, relay_host_id, activity_id + FROM relay_assignment_activity_leases lease + WHERE expires_at <= ? + AND NOT EXISTS ( + SELECT 1 FROM relay_region_rehome_attempts rehome + WHERE rehome.user_id = lease.user_id + AND rehome.relay_host_id = lease.relay_host_id + AND rehome.completed_at IS NULL AND rehome.aborted_at IS NULL + AND lease.activity_id IN ( + 'control-pending:' || CAST(rehome.assignment_epoch AS TEXT), + 'migration:' || CAST(rehome.assignment_epoch AS TEXT) + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM relay_post_drain_migration_pins pin + JOIN relay_assignment_migrations migration + ON migration.user_id = pin.user_id + AND migration.relay_host_id = pin.relay_host_id + AND migration.assignment_epoch = pin.assignment_epoch + WHERE pin.user_id = lease.user_id + AND pin.relay_host_id = lease.relay_host_id + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + AND lease.activity_id IN ( + 'control-pending:' || CAST(pin.assignment_epoch AS TEXT), + 'migration:' || CAST(pin.assignment_epoch AS TEXT) + ) + )`, + [now] + ) + let released = 0 + for (const candidate of candidates) { + let didRelease: boolean + try { + didRelease = await this.database.transaction(async (transaction) => { + const identity = { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + } + // Why: re-check under the canonical assignment-first lock so a concurrent + // renewal wins without cleanup reaping its refreshed lease. + // Several directors sweep the same expired rows. Skip a row another + // director is settling instead of waiting and retrying the transaction. + await this.assignmentRow(transaction, identity, true) + const activityLeases = await this.lockAssignmentActivities(transaction, identity, true) + const lease = activityLeaseById(activityLeases, text(candidate, 'activity_id')) + if (!lease || integer(lease, 'expires_at') > now) return false + await this.lockCellInventory(transaction, true) + await this.removeActivityLease(transaction, identity, lease, now) + return true + }) + } catch (error) { + // Released cell images can still hold their legacy cell-first lock; + // expiry is durable, so a later maintenance sweep can safely retry it. + if (isDatabaseLockUnavailable(error)) continue + throw error + } + if (didRelease) released++ + } + return released + } + + async releaseExpiredActivity(): Promise { + const now = this.now() + try { + return await this.database.transaction(async (transaction) => { + const expired = await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE lease_expires_at <= ? AND + (reserved_controls > 0 OR reserved_splices > 0 OR reserved_invites > 0 OR + pending_installs > 0 OR pending_confirmations > 0 OR migration_leases > 0) + AND NOT EXISTS ( + SELECT 1 FROM relay_region_rehome_attempts rehome + WHERE rehome.user_id = relay_assignments.user_id + AND rehome.relay_host_id = relay_assignments.relay_host_id + AND rehome.completed_at IS NULL AND rehome.aborted_at IS NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM relay_post_drain_migration_pins pin + JOIN relay_assignment_migrations migration + ON migration.user_id = pin.user_id + AND migration.relay_host_id = pin.relay_host_id + AND migration.assignment_epoch = pin.assignment_epoch + WHERE pin.user_id = relay_assignments.user_id + AND pin.relay_host_id = relay_assignments.relay_host_id + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) + ORDER BY user_id, relay_host_id`, + [now], + { failIfUnavailable: true } + ) + if (expired.length > 0) await this.lockCellInventory(transaction, true) + for (const row of expired) { + await this.adjustCellReservation(transaction, text(row, 'cell_id'), -requestUnits(row)) + await transaction.query( + `UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0, + reserved_invites = 0, pending_installs = 0, pending_confirmations = 0, + migration_leases = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [text(row, 'user_id'), text(row, 'relay_host_id')] + ) + } + return expired.length + }) + } catch (error) { + // Aggregate expiry is reconstructible from durable leases; never wait + // long enough to form a mixed-version cell/assignment lock cycle. + if (isDatabaseLockUnavailable(error)) return 0 + throw error + } + } + + async releaseExpiredRegionPreferences(): Promise { + const result = await this.database.query( + `DELETE FROM relay_assignment_region_preferences WHERE observed_at < ?`, + [this.now() - REGION_PREFERENCE_RETENTION_MS] + ) + return integer(result[0]!, 'changes') + } + + private async reconcileReservationAccounting( + sourceCellId: string, + targetCellId: string + ): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + // Reconciliation takes the same assignment→activity→cell order as live + // mutations so correcting drift never races a credential or socket lease. + const assignments = await transaction.queryLocked( + `SELECT assignment.* FROM relay_assignments assignment + WHERE assignment.cell_id IN (?, ?) OR EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases scoped_lease + WHERE scoped_lease.user_id = assignment.user_id + AND scoped_lease.relay_host_id = assignment.relay_host_id + AND scoped_lease.cell_id IN (?, ?) + ) + ORDER BY assignment.user_id, assignment.relay_host_id`, + [sourceCellId, targetCellId, sourceCellId, targetCellId] + ) + const leases = await transaction.queryLocked( + `SELECT lease.* FROM relay_assignment_activity_leases lease + WHERE lease.cell_id IN (?, ?) OR EXISTS ( + SELECT 1 FROM relay_assignments assignment + WHERE assignment.user_id = lease.user_id + AND assignment.relay_host_id = lease.relay_host_id + AND ( + assignment.cell_id IN (?, ?) OR EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases scoped_lease + WHERE scoped_lease.user_id = lease.user_id + AND scoped_lease.relay_host_id = lease.relay_host_id + AND scoped_lease.cell_id IN (?, ?) + ) + ) + ) + ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id`, + [ + sourceCellId, + targetCellId, + sourceCellId, + targetCellId, + sourceCellId, + targetCellId + ] + ) + const cells = await this.lockCellInventory(transaction) + const assignmentKeys = new Set( + assignments.map((row) => + assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id')) + ) + ) + const cellIds = new Set(cells.map((row) => text(row, 'cell_id'))) + const assignmentCounts = new Map< + string, + { counts: Record; leaseExpiresAt: number } + >() + const cellUnits = new Map() + + for (const lease of leases) { + const key = assignmentKey(text(lease, 'user_id'), text(lease, 'relay_host_id')) + if (!assignmentKeys.has(key)) throw new Error('activity_lease_assignment_missing') + const cellId = text(lease, 'cell_id') + if (!cellIds.has(cellId)) throw new Error('activity_lease_cell_missing') + const current = + assignmentCounts.get(key) ?? { + counts: emptyActivityCounts(), + leaseExpiresAt: 0 + } + const kind = activityKind(lease) + current.counts[kind]++ + current.leaseExpiresAt = Math.max(current.leaseExpiresAt, integer(lease, 'expires_at')) + assignmentCounts.set(key, current) + cellUnits.set(cellId, (cellUnits.get(cellId) ?? 0) + integer(lease, 'request_units')) + } + + for (const row of assignments) { + const current = assignmentCounts.get( + assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id')) + ) + const counts = current?.counts ?? emptyActivityCounts() + const differs = (Object.keys(ACTIVITY_COLUMN) as AssignmentActivityKind[]).some( + (kind) => integer(row, ACTIVITY_COLUMN[kind]) !== counts[kind] + ) + const expiryDiffers = + current !== undefined && integer(row, 'lease_expires_at') !== current.leaseExpiresAt + if (!differs && !expiryDiffers) continue + await transaction.query( + `UPDATE relay_assignments SET reserved_controls = ?, reserved_splices = ?, + reserved_invites = ?, pending_installs = ?, pending_confirmations = ?, + migration_leases = ?, lease_expires_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [ + counts.control, + counts.splice, + counts.invite, + counts.install, + counts.confirmation, + counts.migration, + current?.leaseExpiresAt ?? integer(row, 'lease_expires_at'), + text(row, 'user_id'), + text(row, 'relay_host_id') + ] + ) + } + + for (const row of cells.filter((cell) => + [sourceCellId, targetCellId].includes(text(cell, 'cell_id')) + )) { + const cellId = text(row, 'cell_id') + const expected = cellUnits.get(cellId) ?? 0 + if (expected > integer(row, 'capacity_requests')) { + throw new Error('relay_capacity_exhausted') + } + if (integer(row, 'reserved_requests') === expected) continue + await transaction.query( + `UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`, + [expected, now, cellId] + ) + } + }) + } + + private async lockCellInventory( + database: RelayDatabase, + failIfUnavailable = false + ): Promise { + // Every capacity-changing assignment takes the tiny cell inventory in one + // order; dynamically locking only the selected target allowed cross-cell cycles. + return await database.queryLocked( + `SELECT * FROM relay_cells ORDER BY cell_id ASC`, + [], + { failIfUnavailable } + ) + } + + private async lockGeneralCellInventory( + database: RelayDatabase, + failIfUnavailable = false + ): Promise { + return await database.queryLocked( + `SELECT * FROM relay_cells + WHERE cell_id IN ( + SELECT cell_id FROM relay_cell_admission WHERE admission_state = 'general' + ) + ORDER BY cell_id ASC`, + [], + { failIfUnavailable } + ) + } + + private async leastLoadedCell( + database: RelayDatabase, + lockedCells: SqlRow[] | undefined, + preferredRegion: RelayRegion + ): Promise { + const rows = lockedCells ?? (await this.lockCellInventory(database)) + const regions = new Map( + (await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [ + text(row, 'cell_id'), + relayRegion(row, 'region') + ]) + ) + const admission = await cellAdmissionStates(database) + const runtimeLoad = this.requireLiveCells + ? new Map( + ( + await database.query( + `SELECT cell_id, observed_requests FROM relay_cell_runtime + WHERE ready = ? AND last_heartbeat_at > ?`, + [1, this.now() - this.heartbeatTtlMs] + ) + ).map((row) => [text(row, 'cell_id'), integer(row, 'observed_requests')]) + ) + : null + const connectionHeadroom = await this.connectionHeadroomByCell(database) + const candidates = rows.filter((row) => { + const cellId = text(row, 'cell_id') + return ( + admission.get(cellId) === 'general' && + integer(row, 'reserved_requests') < integer(row, 'capacity_requests') && + (!runtimeLoad || runtimeLoad.has(cellId)) && + connectionHeadroom.get(cellId) !== false + ) + }) + candidates.sort((left, right) => { + const leftLoad = + integer(left, 'reserved_requests') + + (runtimeLoad?.get(text(left, 'cell_id')) ?? integer(left, 'observed_requests')) + const rightLoad = + integer(right, 'reserved_requests') + + (runtimeLoad?.get(text(right, 'cell_id')) ?? integer(right, 'observed_requests')) + const loadDifference = + leftLoad / integer(left, 'capacity_requests') - + rightLoad / integer(right, 'capacity_requests') + return loadDifference || text(left, 'cell_id').localeCompare(text(right, 'cell_id')) + }) + const preferred = candidates.filter( + (candidate) => + (regions.get(text(candidate, 'cell_id')) ?? RELAY_DEFAULT_REGION) === preferredRegion + ) + const selected = preferred[0] ?? candidates[0] + return selected + ? cell(selected, regions.get(text(selected, 'cell_id')) ?? RELAY_DEFAULT_REGION) + : null + } + + async regionCatalog(): Promise { + const rows = await this.database.query( + `SELECT cell.cell_id, cell.cell_url, region.region + FROM relay_cells cell + LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id + JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id + JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id + WHERE cell.enabled = 1 + AND admission.admission_state = 'general' + AND runtime.ready = ? AND runtime.last_heartbeat_at > ? + ORDER BY cell.cell_id ASC`, + [1, this.now() - this.heartbeatTtlMs] + ) + const origins = new Map() + for (const row of rows) { + const region = optionalRelayRegion(row, 'region') ?? RELAY_DEFAULT_REGION + const current = origins.get(region) ?? [] + if (current.length < 2) current.push(text(row, 'cell_url')) + origins.set(region, current) + } + return RELAY_REGIONS.flatMap((region) => { + const probeOrigins = origins.get(region) + return probeOrigins?.length ? [{ region, probeOrigins }] : [] + }) + } + + private async recordRegionPreference( + database: RelayDatabase, + identity: AssignmentIdentity, + preferredRegion: RelayRegion | undefined, + observedAt: number + ): Promise { + if (!preferredRegion) return + await database.query( + `INSERT INTO relay_assignment_region_preferences + (user_id, relay_host_id, preferred_region, observed_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (user_id, relay_host_id) DO UPDATE SET + preferred_region = CASE + WHEN excluded.observed_at >= relay_assignment_region_preferences.observed_at + THEN excluded.preferred_region + ELSE relay_assignment_region_preferences.preferred_region + END, + observed_at = CASE + WHEN excluded.observed_at >= relay_assignment_region_preferences.observed_at + THEN excluded.observed_at + ELSE relay_assignment_region_preferences.observed_at + END`, + [identity.userId, identity.relayHostId, preferredRegion, observedAt] + ) + } + + private async cellRegion(database: RelayDatabase, cellId: string): Promise { + const row = ( + await database.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [cellId]) + )[0] + return row ? relayRegion(row, 'region') : RELAY_DEFAULT_REGION + } + + private async connectionHeadroomByCell( + database: RelayDatabase + ): Promise> { + const now = this.now() + const rows = await database.query(ASSIGNMENT_CONNECTION_HEADROOM_QUERY) + return new Map( + rows.map((row) => { + const heartbeat = optionalInteger(row, 'last_heartbeat_at') + const hasFreshTelemetry = + heartbeat !== undefined && + heartbeat > now - this.heartbeatTtlMs && + optionalText(row, 'connection_incarnation') === + optionalText(row, 'current_incarnation') + const hasHeadroom = + hasFreshTelemetry && + integer(row, 'enforced_connection_units') + + integer(row, 'outstanding_reservations') + + integer(row, 'unobserved_bound') < + integer(row, 'hard_cap') - + RELAY_ADMISSION_BUDGETS.reservedHostControls + return [text(row, 'cell_id'), hasHeadroom] as const + }) + ) + } + + private async assertCellConnectionHeadroom( + database: RelayDatabase, + cellId: string + ): Promise { + if (!(await this.cellHasConnectionHeadroom(database, cellId))) { + throw new Error('relay_connection_headroom_exhausted') + } + } + + private async cellHasConnectionHeadroom( + database: RelayDatabase, + cellId: string + ): Promise { + return (await this.connectionHeadroomByCell(database)).get(cellId) !== false + } + + private async cellIsLive( + database: RelayDatabase, + cellId: string, + now: number + ): Promise { + if (!this.requireLiveCells) return true + const rows = await database.query( + `SELECT cell.cell_id FROM relay_cells cell + JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id + WHERE cell.cell_id = ? AND runtime.ready = ? AND runtime.last_heartbeat_at > ?`, + [cellId, 1, now - this.heartbeatTtlMs] + ) + return rows.length === 1 + } + + private async cellHasActiveFence(cellId: string): Promise { + const rows = await this.database.query( + `SELECT fence.cell_id FROM relay_cell_fences fence + JOIN relay_cells cell ON cell.cell_id = fence.cell_id + JOIN relay_cell_runtime runtime ON runtime.cell_id = fence.cell_id + WHERE fence.cell_id = ? AND cell.enabled = 0 + AND fence.cell_incarnation = runtime.cell_incarnation + AND fence.attested_at >= runtime.last_heartbeat_at + AND fence.expires_at > ?`, + [cellId, this.now()] + ) + return rows.length === 1 + } + + private async recordCommittedCellFence( + transaction: RelayDatabase, + cellId: string, + cellIncarnation: string, + attemptId: string, + attestedAt: number, + expiresAt: number + ): Promise { + await transaction.query( + `INSERT INTO relay_cell_committed_fences + (cell_id, attempt_id, cell_incarnation, attested_at, expires_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (cell_id) DO UPDATE SET + attempt_id = excluded.attempt_id, + cell_incarnation = excluded.cell_incarnation, + attested_at = excluded.attested_at, + expires_at = excluded.expires_at`, + [cellId, attemptId, cellIncarnation, attestedAt, expiresAt] + ) + } + + private async cellHasCommittedFence( + database: RelayDatabase, + cellId: string, + now: number + ): Promise { + const rows = await database.query( + `SELECT committed.cell_id + FROM relay_cell_committed_fences committed + JOIN relay_cell_fence_attempts attempt + ON attempt.attempt_id = committed.attempt_id + JOIN relay_cell_fences fence ON fence.cell_id = committed.cell_id + JOIN relay_cell_runtime runtime ON runtime.cell_id = committed.cell_id + JOIN relay_cells cell ON cell.cell_id = committed.cell_id + WHERE committed.cell_id = ? + AND attempt.completed_at IS NOT NULL + AND attempt.aborted_at IS NULL + AND cell.enabled = 0 + AND committed.cell_incarnation = runtime.cell_incarnation + AND fence.cell_incarnation = committed.cell_incarnation + AND committed.attested_at >= runtime.last_heartbeat_at + AND committed.expires_at > ? + AND fence.expires_at > ?`, + [cellId, now, now] + ) + return rows.length === 1 + } + + private async deadCellRequiresCommittedFence( + database: RelayDatabase, + identity: AssignmentIdentity, + cellId: string, + assignmentEpoch: number + ): Promise { + const row = ( + await database.query( + `SELECT + CASE WHEN EXISTS ( + SELECT 1 FROM relay_cell_connection_limits limits + WHERE limits.cell_id = ? + ) THEN 1 ELSE 0 END AS capped, + CASE WHEN EXISTS ( + SELECT 1 FROM relay_post_drain_migration_pins pin + JOIN relay_assignment_migrations migration + ON migration.user_id = pin.user_id + AND migration.relay_host_id = pin.relay_host_id + AND migration.assignment_epoch = pin.assignment_epoch + WHERE pin.user_id = ? AND pin.relay_host_id = ? + AND pin.assignment_epoch = ? + AND pin.target_cell_id = ? + AND migration.completed_at IS NULL + AND migration.aborted_at IS NULL + ) THEN 1 ELSE 0 END AS pinned`, + [cellId, identity.userId, identity.relayHostId, assignmentEpoch, cellId] + ) + )[0] + if (!row) return false + return integer(row, 'capped') === 1 || integer(row, 'pinned') === 1 + } + + private async assertDrainCellGeneration( + transaction: RelayDatabase, + cellId: string, + cellIncarnation: string + ): Promise { + const cell = ( + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId]) + )[0] + const runtime = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [ + cellId + ]) + )[0] + if (!cell || integer(cell, 'enabled') !== 0) { + throw new Error('drain_attempt_admission_enabled') + } + if (!runtime || text(runtime, 'cell_incarnation') !== cellIncarnation) { + throw new Error('drain_attempt_generation_mismatch') + } + } + + private async requireCellFence( + transaction: RelayDatabase, + cellId: string, + runtime: SqlRow | undefined, + now: number + ): Promise { + const fence = ( + await transaction.queryLocked(`SELECT * FROM relay_cell_fences WHERE cell_id = ?`, [ + cellId + ]) + )[0] + if ( + !fence || + !runtime || + text(fence, 'cell_incarnation') !== text(runtime, 'cell_incarnation') || + integer(fence, 'attested_at') < integer(runtime, 'last_heartbeat_at') || + integer(fence, 'expires_at') <= now + ) { + throw new Error('cell_fence_attestation_missing') + } + } + + private async activeCellMigrations(sourceCellId: string, targetCellId: string): Promise { + const targetRuntimeSafety = this.requireLiveCells + ? `AND EXISTS ( + SELECT 1 FROM relay_cell_runtime target_runtime + WHERE target_runtime.cell_id = migration.target_cell_id + AND lease.updated_at >= target_runtime.started_at + )` + : '' + return await this.database.query( + `SELECT migration.*, assignment.cell_id AS current_cell_id, + assignment.assignment_epoch AS current_assignment_epoch, + COALESCE(( + SELECT SUM(source_lease.request_units) + FROM relay_assignment_activity_leases source_lease + WHERE source_lease.user_id = migration.user_id + AND source_lease.relay_host_id = migration.relay_host_id + AND source_lease.cell_id = migration.source_cell_id + ), 0) AS source_activity_units, + CASE WHEN EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases lease + WHERE lease.user_id = migration.user_id + AND lease.relay_host_id = migration.relay_host_id + AND lease.cell_id = migration.target_cell_id + AND lease.activity_kind = 'control' + AND lease.activity_id NOT LIKE 'control-pending:%' + ) THEN 1 ELSE 0 END AS target_control_active, + CASE WHEN EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases lease + WHERE lease.user_id = migration.user_id + AND lease.relay_host_id = migration.relay_host_id + AND lease.cell_id = migration.target_cell_id + AND lease.activity_kind = 'control' + AND lease.activity_id NOT LIKE 'control-pending:%' + AND lease.expires_at > ? + ${targetRuntimeSafety} + ) THEN 1 ELSE 0 END AS target_control_current + FROM relay_assignment_migrations migration + LEFT JOIN relay_assignments assignment + ON assignment.user_id = migration.user_id + AND assignment.relay_host_id = migration.relay_host_id + WHERE migration.source_cell_id = ? AND migration.target_cell_id = ? + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + ORDER BY migration.user_id, migration.relay_host_id`, + [this.now(), sourceCellId, targetCellId] + ) + } + + private async insertPendingControlLease( + database: RelayDatabase, + identity: AssignmentIdentity, + cellId: string, + assignmentEpoch: number, + now: number + ): Promise { + const timeoutAt = now + ASSIGNMENT_LIMITS.activityLeaseMs + await database.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (user_id, relay_host_id, activity_id) DO UPDATE SET + expires_at = excluded.expires_at, updated_at = excluded.updated_at`, + [ + identity.userId, + identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + 'control', + cellId, + 1, + timeoutAt, + now + ] + ) + await this.insertControlConnectionReservation( + database, + identity, + cellId, + assignmentEpoch, + timeoutAt, + now + ) + } + + private async insertControlConnectionReservation( + database: RelayDatabase, + identity: AssignmentIdentity, + cellId: string, + assignmentEpoch: number, + timeoutAt: number, + now: number + ): Promise { + const existing = await database.queryLocked( + `SELECT reservation_id + FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND cell_id = ? AND state IN ('reserved', 'late-arrival-debt') + AND claim_activity_id IS NULL + ORDER BY created_at ASC, reservation_id ASC + LIMIT 1`, + [identity.userId, identity.relayHostId, assignmentEpoch, cellId] + ) + if (existing.length > 0) return + const reservationId = randomUUID() + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + SELECT ?, ?, ?, ?, ?, ?, 'reserved', NULL, NULL, ?, ?, NULL, NULL, ? + FROM relay_cell_connection_limits WHERE cell_id = ?`, + [ + reservationId, + reservationId, + identity.userId, + identity.relayHostId, + assignmentEpoch, + cellId, + now, + timeoutAt, + now, + cellId + ] + ) + } + + private async refreshPendingControlReservation( + database: RelayDatabase, + identity: AssignmentIdentity, + cellId: string, + assignmentEpoch: number, + timeoutAt: number, + now: number + ): Promise { + await database.query( + `UPDATE relay_control_connection_reservations + SET state = 'reserved', timeout_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND cell_id = ? AND state IN ('reserved', 'late-arrival-debt') + AND claim_activity_id IS NULL`, + [ + timeoutAt, + now, + identity.userId, + identity.relayHostId, + assignmentEpoch, + cellId + ] + ) + } + + private async claimControlConnectionReservation( + database: RelayDatabase, + identity: AssignmentIdentity, + cellId: string, + assignmentEpoch: number, + activityId: string, + inclusionWatermark: number | undefined, + now: number + ): Promise { + await database.query( + `UPDATE relay_control_connection_reservations + SET claim_activity_id = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND cell_id = ? AND state = 'claimed' + AND claim_activity_id IS NOT NULL AND claim_activity_id <> ?`, + [ + activityId, + now, + identity.userId, + identity.relayHostId, + assignmentEpoch, + cellId, + activityId + ] + ) + // A released row's claim_activity_id is history, not a live claim: a control that + // reconnects under the same generation would otherwise leave its fresh reservation + // unclaimable, decaying through debt while the connection is already enforced. + const alreadyClaimed = await database.queryLocked( + `SELECT reservation_id FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND cell_id = ? AND claim_activity_id = ? AND state <> 'released' + ORDER BY created_at ASC, reservation_id ASC`, + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + cellId, + activityId + ] + ) + if (alreadyClaimed.length > 0) return + const reservation = ( + await database.queryLocked( + `SELECT * FROM relay_control_connection_reservations + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? + AND cell_id = ? AND state IN ('reserved', 'late-arrival-debt') + AND claim_activity_id IS NULL + ORDER BY created_at ASC, reservation_id ASC`, + [identity.userId, identity.relayHostId, assignmentEpoch, cellId] + ) + )[0] + if (!reservation) return + await database.query( + `UPDATE relay_control_connection_reservations + SET state = 'claimed', inclusion_watermark = ?, claim_activity_id = ?, + claimed_at = ?, updated_at = ? + WHERE reservation_id = ?`, + [ + inclusionWatermark, + activityId, + now, + now, + text(reservation, 'reservation_id') + ] + ) + } + + private async releaseSupersededControlConnectionReservations( + database: RelayDatabase, + identity: AssignmentIdentity, + cellId: string, + assignmentEpoch: number, + now: number + ): Promise { + await database.query( + `UPDATE relay_control_connection_reservations + SET state = 'released', released_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND cell_id = ? + AND assignment_epoch = ? AND state <> 'released'`, + [ + now, + now, + identity.userId, + identity.relayHostId, + cellId, + assignmentEpoch + ] + ) + } + + private async assignmentRow( + database: RelayDatabase, + identity: AssignmentIdentity, + failIfUnavailable = false + ): Promise { + return ( + await database.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId], + { failIfUnavailable } + ) + )[0] + } + + private async lockAssignmentActivities( + database: RelayDatabase, + identity: AssignmentIdentity, + failIfUnavailable = false + ): Promise { + // Lease rows always precede the globally ordered capacity rows. + return await database.queryLocked( + `SELECT * FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC`, + [identity.userId, identity.relayHostId], + { failIfUnavailable } + ) + } + + private async removeActivityLease( + database: RelayDatabase, + identity: AssignmentIdentity, + lease: SqlRow, + now: number + ): Promise { + const kind = activityKind(lease) + await this.adjustCellReservation( + database, + text(lease, 'cell_id'), + -integer(lease, 'request_units') + ) + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, text(lease, 'activity_id')] + ) + await this.adjustActivityCount(database, identity, kind, -1, now, now) + } + + private async removeSupersededSameCellControls( + database: RelayDatabase, + identity: AssignmentIdentity, + leases: SqlRow[], + cellId: string, + retainedActivityId: string, + now: number + ): Promise { + const superseded = leases.filter( + (lease) => + activityKind(lease) === 'control' && + text(lease, 'cell_id') === cellId && + text(lease, 'activity_id') !== retainedActivityId + ) + if (superseded.length === 0) return + if ( + superseded.some( + (lease) => integer(lease, 'request_units') !== ACTIVITY_REQUEST_UNITS.control + ) + ) { + throw new Error('activity_lease_shape_mismatch') + } + const cells = await this.lockCellInventory(database) + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control' + AND cell_id = ? AND activity_id <> ?`, + [identity.userId, identity.relayHostId, cellId, retainedActivityId] + ) + const remainingControls = + leases.filter((lease) => activityKind(lease) === 'control').length - superseded.length + await database.query( + `UPDATE relay_assignments SET reserved_controls = ?, last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [remainingControls, now, identity.userId, identity.relayHostId] + ) + const cellUnitsRow = ( + await database.query( + `SELECT COALESCE(SUM(request_units), 0) AS request_units + FROM relay_assignment_activity_leases WHERE cell_id = ?`, + [cellId] + ) + )[0]! + const cellRow = cells.find((cell) => text(cell, 'cell_id') === cellId) + const cellUnits = integer(cellUnitsRow, 'request_units') + if (!cellRow) throw new Error('assigned_cell_missing') + if (cellUnits > integer(cellRow, 'capacity_requests')) { + throw new Error('relay_capacity_exhausted') + } + await database.query( + `UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`, + [cellUnits, now, cellId] + ) + } + + private async adjustActivityCount( + database: RelayDatabase, + identity: AssignmentIdentity, + kind: AssignmentActivityKind, + delta: 1 | -1, + leaseExpiresAt: number, + now: number + ): Promise { + const column = ACTIVITY_COLUMN[kind] + await database.query( + `UPDATE relay_assignments SET ${column} = + CASE WHEN ${column} + ? < 0 THEN 0 ELSE ${column} + ? END, + lease_expires_at = + CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, + last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [delta, delta, leaseExpiresAt, leaseExpiresAt, now, identity.userId, identity.relayHostId] + ) + } + + private async touchAssignment( + database: RelayDatabase, + identity: AssignmentIdentity, + leaseExpiresAt: number, + now: number + ): Promise { + await database.query( + `UPDATE relay_assignments SET lease_expires_at = + CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, + last_activity_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [leaseExpiresAt, leaseExpiresAt, now, identity.userId, identity.relayHostId] + ) + } + + private async adjustCellReservation( + database: RelayDatabase, + cellId: string, + delta: number + ): Promise { + const row = (await database.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId]))[0] + if (!row) throw new Error('assigned_cell_missing') + const next = integer(row, 'reserved_requests') + delta + if (next > integer(row, 'capacity_requests')) throw new Error('relay_capacity_exhausted') + await database.query( + `UPDATE relay_cells SET reserved_requests = + CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, + updated_at = ? WHERE cell_id = ?`, + [delta, delta, this.now(), cellId] + ) + } + + private async adjustCellReservationAtomically( + database: RelayDatabase, + cellId: string, + delta: number + ): Promise { + const rows = await database.query( + `UPDATE relay_cells SET reserved_requests = + CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, + updated_at = ? + WHERE cell_id = ? + AND (? <= 0 OR reserved_requests + ? <= capacity_requests) + RETURNING cell_id`, + [delta, delta, this.now(), cellId, delta, delta] + ) + if (rows.length > 0) return + const cell = ( + await database.query(`SELECT cell_id FROM relay_cells WHERE cell_id = ?`, [cellId]) + )[0] + if (!cell) throw new Error('assigned_cell_missing') + throw new Error('relay_capacity_exhausted') + } + + private result( + identity: AssignmentIdentity, + row: SqlRow, + cellRow: CellRow, + leaseExpiresAt: number + ): RelayAssignment { + return { + ...identity, + ...cellRow, + assignmentEpoch: integer(row, 'assignment_epoch'), + leaseExpiresAt + } + } +} + +type CellRow = { cellId: string; cellUrl: string; region: RelayRegion } + +function cell(row: SqlRow, region: RelayRegion): CellRow { + return { cellId: text(row, 'cell_id'), cellUrl: text(row, 'cell_url'), region } +} + +function relayRegion(row: SqlRow, field: string): RelayRegion { + const value = text(row, field) + if (!RELAY_REGIONS.includes(value as RelayRegion)) throw new Error(`invalid region field ${field}`) + return value as RelayRegion +} + +function optionalRelayRegion(row: SqlRow, field: string): RelayRegion | undefined { + const value = optionalText(row, field) + if (value === undefined) return undefined + if (!RELAY_REGIONS.includes(value as RelayRegion)) throw new Error(`invalid region field ${field}`) + return value as RelayRegion +} + +function activityLeaseById(rows: SqlRow[], activityId: string): SqlRow | undefined { + return rows.find((row) => text(row, 'activity_id') === activityId) +} + +// Why: mid-rehome a host legitimately holds the source control plus the +// target's pending control, so only the lease rows — never the counter a grant +// would overwrite — can say whether this cell is already reserved for it. +function holdsControlLease( + rows: SqlRow[], + cellId: string, + assignmentEpoch: number +): boolean { + const pendingId = pendingControlActivityId(assignmentEpoch) + return rows.some((row) => { + if (activityKind(row) !== 'control' || text(row, 'cell_id') !== cellId) return false + const activityId = text(row, 'activity_id') + return activityId === pendingId || !activityId.startsWith('control-pending:') + }) +} + +function activityCounts(rows: SqlRow[]): Record { + const counts = emptyActivityCounts() + for (const row of rows) counts[activityKind(row)]++ + return counts +} + +function activityUnitsForCell(rows: SqlRow[], cellId: string): number { + return rows.reduce( + (total, row) => + text(row, 'cell_id') === cellId ? total + integer(row, 'request_units') : total, + 0 + ) +} + +function activity(row: SqlRow) { + return { + relayHostId: text(row, 'relay_host_id'), + cellId: text(row, 'cell_id'), + assignmentEpoch: integer(row, 'assignment_epoch'), + leaseExpiresAt: integer(row, 'lease_expires_at'), + lastActivityAt: integer(row, 'last_activity_at'), + reservedControls: integer(row, 'reserved_controls'), + reservedSplices: integer(row, 'reserved_splices'), + reservedInvites: integer(row, 'reserved_invites'), + pendingInstalls: integer(row, 'pending_installs'), + pendingConfirmations: integer(row, 'pending_confirmations'), + migrationLeases: integer(row, 'migration_leases') + } +} + +function requestUnits(row: SqlRow): number { + return ( + integer(row, 'reserved_controls') + + 2 * integer(row, 'reserved_splices') + + integer(row, 'reserved_invites') + + integer(row, 'pending_installs') + + integer(row, 'pending_confirmations') + + integer(row, 'migration_leases') + ) +} + +function integer(row: SqlRow, field: string): number { + const value = Number(row[field]) + if (!Number.isSafeInteger(value)) throw new Error(`invalid_${field}`) + return value +} + +function text(row: SqlRow, field: string): string { + const value = row[field] + if (typeof value !== 'string') throw new Error(`invalid_${field}`) + return value +} + +function cellFenceAttempt(row: SqlRow): CellFenceAttempt { + const environment = text(row, 'environment') + if (!['staging', 'production'].includes(environment)) { + throw new Error('invalid_environment') + } + return { + attemptId: text(row, 'attempt_id'), + environment: environment as CellFenceAttempt['environment'], + cellId: text(row, 'cell_id'), + cellIncarnation: text(row, 'cell_incarnation'), + migName: text(row, 'mig_name'), + instanceGroup: text(row, 'instance_group'), + generationIdentity: text(row, 'generation_identity'), + fenceCommit: text(row, 'fence_commit'), + planSha256: text(row, 'plan_sha256'), + planObjectName: text(row, 'plan_object_name'), + planObjectGeneration: optionalText(row, 'plan_object_generation'), + varFileSha256: text(row, 'var_file_sha256'), + terraformStateLineage: text(row, 'terraform_state_lineage'), + terraformStateSerial: integer(row, 'terraform_state_serial'), + terraformStateObjectGeneration: text( + row, + 'terraform_state_object_generation' + ), + terraformStateObjectSha256: text(row, 'terraform_state_object_sha256'), + requestReason: text(row, 'request_reason'), + gceOperation: optionalText(row, 'gce_operation'), + createdAt: integer(row, 'created_at'), + expiresAt: integer(row, 'expires_at'), + applyStartedAt: optionalInteger(row, 'apply_started_at'), + completedAt: optionalInteger(row, 'completed_at'), + abortedAt: optionalInteger(row, 'aborted_at') + } +} + +function cellFenceApplyInvocation(row: SqlRow): CellFenceApplyInvocation { + return { + invocationId: text(row, 'invocation_id'), + requestReason: text(row, 'request_reason'), + startedAt: integer(row, 'started_at'), + gceOperation: optionalText(row, 'gce_operation') + } +} + +function assertCellFenceAttemptBase( + row: SqlRow, + input: CellFenceAttemptEvidence +): void { + const fields: [keyof CellFenceAttemptEvidence, string][] = [ + ['attemptId', 'attempt_id'], + ['environment', 'environment'], + ['cellId', 'cell_id'], + ['cellIncarnation', 'cell_incarnation'], + ['migName', 'mig_name'], + ['instanceGroup', 'instance_group'], + ['generationIdentity', 'generation_identity'], + ['fenceCommit', 'fence_commit'], + ['planSha256', 'plan_sha256'], + ['planObjectName', 'plan_object_name'], + ['varFileSha256', 'var_file_sha256'], + ['terraformStateLineage', 'terraform_state_lineage'], + ['terraformStateObjectGeneration', 'terraform_state_object_generation'], + ['terraformStateObjectSha256', 'terraform_state_object_sha256'], + ['requestReason', 'request_reason'] + ] + if ( + fields.some(([inputField, rowField]) => input[inputField] !== text(row, rowField)) || + input.terraformStateSerial !== integer(row, 'terraform_state_serial') + ) { + throw new Error('cell_fence_attempt_evidence_mismatch') + } +} + +function assertCellFenceAttemptEvidence( + row: SqlRow, + input: CellFenceAttemptEvidence +): void { + assertCellFenceAttemptBase(row, input) + if ( + !input.planObjectGeneration || + input.planObjectGeneration !== optionalText(row, 'plan_object_generation') + ) { + throw new Error('cell_fence_attempt_evidence_mismatch') + } +} + +function assertActiveCellFenceAttempt( + row: SqlRow, + input: CellFenceAttemptEvidence, + now: number +): void { + assertCellFenceAttemptEvidence(row, input) + if ( + integer(row, 'expires_at') <= now && + optionalInteger(row, 'apply_started_at') === undefined + ) { + throw new Error('cell_fence_attempt_expired') + } + if (row.completed_at !== null) throw new Error('cell_fence_attempt_completed') + if (row.aborted_at !== null) throw new Error('cell_fence_attempt_aborted') +} + +async function lockedCellFenceAttempt( + transaction: RelayDatabase, + attemptId: string +): Promise { + const row = ( + await transaction.queryLocked( + `${CELL_FENCE_ATTEMPT_SELECT} WHERE attempts.attempt_id = ?`, + [attemptId] + ) + )[0] + if (!row) throw new Error('cell_fence_attempt_not_found') + return row +} + +function pendingControlActivityId(assignmentEpoch: number): string { + return `control-pending:${assignmentEpoch}` +} + +function validateActivityId(activityId: string): void { + if (!activityId || activityId.length > 256) throw new Error('invalid_activity_id') +} + +function activityKind(row: SqlRow): AssignmentActivityKind { + const value = text(row, 'activity_kind') + if (!(value in ACTIVITY_REQUEST_UNITS)) throw new Error('invalid_activity_kind') + return value as AssignmentActivityKind +} + +function migrationActivityId(assignmentEpoch: number): string { + return `migration:${assignmentEpoch}` +} + +function isIncompleteMigration(error: unknown): boolean { + return ( + error instanceof Error && + [ + 'migration_target_not_registered', + 'migration_source_still_active', + 'migration_target_not_active', + 'migration_assignment_mismatch', + 'migration_source_admission_changed', + 'migration_target_admission_changed', + 'migration_source_runtime_not_quiescent', + 'migration_source_runtime_not_dead', + 'migration_target_runtime_not_ready', + 'migration_cell_inventory_busy', + 'migration_activity_accounting_mismatch', + 'migration_activity_topology_mismatch', + 'migration_activity_lease_shape_mismatch', + 'migration_cell_reservation_accounting_mismatch', + 'cell_fence_attestation_missing' + ].includes(error.message) + ) +} + +function isMissingMigration(error: unknown): boolean { + return error instanceof Error && error.message === 'migration_not_found' +} + +function isDatabaseLockUnavailable(error: unknown): boolean { + return error instanceof Error && error.message === 'database_lock_unavailable' +} + +function isDatabaseLockTimeout(error: unknown): boolean { + return String((error as { code?: unknown }).code) === '55P03' +} + +async function waitForAssignmentLockRetry(deadline: number): Promise { + const remainingMs = Math.max(0, deadline - Date.now()) + const maxDelayMs = Math.min(ASSIGNMENT_LOCK_RETRY_MAX_DELAY_MS, remainingMs) + const delayMs = Math.floor(Math.random() * (maxDelayMs + 1)) + await new Promise((resolve) => setTimeout(resolve, delayMs)) +} + +function migration(identity: AssignmentIdentity, row: SqlRow): RelayAssignmentMigration { + const targetRegisteredAt = optionalInteger(row, 'target_registered_at') + return { + ...identity, + sourceCellId: text(row, 'source_cell_id'), + targetCellId: text(row, 'target_cell_id'), + previousEpoch: integer(row, 'previous_epoch'), + assignmentEpoch: integer(row, 'assignment_epoch'), + expiresAt: integer(row, 'expires_at'), + ...(targetRegisteredAt === undefined ? {} : { targetRegisteredAt }) + } +} + +// Attempt ids are server-minted UUIDs and this codebase's invariant messages +// are snake_case slugs; anything else could carry secrets and logs redacted. +function warnRegionalRehomeCandidateFailure( + operation: 'complete' | 'abort', + attemptId: string, + error: unknown +): void { + const message = error instanceof Error ? error.message : '' + console.warn( + JSON.stringify({ + event: 'orca_relay_regional_rehome_candidate_failed', + operation, + attemptId, + reason: /^[a-z0-9_]{1,64}$/.test(message) ? message : 'redacted' + }) + ) +} + +// The attempt id is the only field: counts would say which host holds what. +function noteRegionalRehomeActivityCountsRepaired(attemptId: string): void { + console.warn( + JSON.stringify({ + event: 'orca_relay_regional_rehome_activity_counts_repaired', + attemptId + }) + ) +} + +function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt { + return { + attemptId: text(row, 'attempt_id'), + userId: text(row, 'user_id'), + relayHostId: text(row, 'relay_host_id'), + preferredRegion: 'asia-east2', + sourceCellId: text(row, 'source_cell_id'), + sourceCellUrl: text(row, 'source_cell_url'), + sourceCellIncarnation: text(row, 'source_cell_incarnation'), + targetCellId: text(row, 'target_cell_id'), + targetCellIncarnation: text(row, 'target_cell_incarnation'), + previousEpoch: integer(row, 'previous_epoch'), + assignmentEpoch: integer(row, 'assignment_epoch'), + drainGraceMs: integer(row, 'drain_grace_ms'), + sendAttempts: integer(row, 'send_attempts') + } +} + +function regionalRehomeControl(row: SqlRow): RegionalRehomeControl { + return { + generation: integer(row, 'generation'), + enabled: integer(row, 'enabled') === 1, + observationStartedAt: integer(row, 'observation_started_at'), + notBefore: integer(row, 'not_before'), + ratePerMinute: integer(row, 'rate_per_minute'), + preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'), + drainGraceMs: integer(row, 'drain_grace_ms') + } +} + +function cleanRegionalRehomeSafety(now: number): RegionalRehomeSafetySnapshot { + return { + observedAt: now, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } +} + +function regionalRehomeFleetSafetyFromInventory(input: { + cells: SqlRow[] + admission: ReadonlyMap + regions: ReadonlyMap + runtimes: SqlRow[] + capabilities: SqlRow[] + safetyRows: SqlRow[] + now: number + heartbeatTtlMs: number +}): RegionalRehomeFleetSafety { + const capabilities = new Map( + input.capabilities.map((row) => [text(row, 'cell_id'), row]) + ) + const runtimes = new Map(input.runtimes.map((row) => [text(row, 'cell_id'), row])) + const safetyRows = new Map( + input.safetyRows.map((row) => [text(row, 'cell_id'), row]) + ) + const required = input.cells.filter((row) => { + const cellId = text(row, 'cell_id') + const capability = capabilities.get(cellId) + return ( + integer(row, 'enabled') === 1 && + input.admission.get(cellId) === 'general' && + (input.regions.get(cellId) === 'asia-east2' || + (input.regions.get(cellId) === RELAY_DEFAULT_REGION && + capability !== undefined && + integer(capability, 'regional_rehome_protocol') >= 1)) + ) + }) + const valid = required.flatMap((row) => { + const cellId = text(row, 'cell_id') + const runtime = runtimes.get(cellId) + const safety = safetyRows.get(cellId) + if ( + !runtime || + integer(runtime, 'ready') !== 1 || + integer(runtime, 'last_heartbeat_at') <= input.now - input.heartbeatTtlMs || + !safety || + text(safety, 'cell_incarnation') !== text(runtime, 'cell_incarnation') || + integer(safety, 'observed_at') <= input.now - 60_000 + ) { + return [] + } + return [safety] + }) + const missingCells = required.length === 0 ? 1 : required.length - valid.length + return { + requiredCells: required.length, + missingCells, + observedAt: + missingCells > 0 + ? 0 + : Math.min(...valid.map((row) => integer(row, 'observed_at'))), + sqlFailures: valid.reduce((total, row) => total + integer(row, 'sql_failures'), 0), + reconnects: valid.reduce((total, row) => total + integer(row, 'reconnects'), 0), + maxReconnects: Math.max(0, ...valid.map((row) => integer(row, 'reconnects'))), + controlActivityRecoveryFailures: valid.reduce( + (total, row) => total + integer(row, 'control_activity_recovery_failures'), + 0 + ), + databasePoolWaiting: Math.max( + 0, + ...valid.map((row) => integer(row, 'database_pool_waiting')) + ), + databasePoolWaitersMax: Math.max( + 0, + ...valid.map((row) => integer(row, 'database_pool_waiters_max')) + ), + databasePoolWaitMsMax: Math.max( + 0, + ...valid.map((row) => integer(row, 'database_pool_wait_ms_max')) + ) + } +} + +function regionalRehomeFleetSafetyFailure( + processSafety: RegionalRehomeSafetySnapshot, + fleetSafety: RegionalRehomeFleetSafety, + now: number +): string | null { + if (fleetSafety.maxReconnects > REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT) { + return 'elevated_reconnects' + } + return regionalRehomeSafetyFailure( + combineRegionalRehomeSafety(processSafety, fleetSafety), + now, + fleetSafety.requiredCells + ) +} + +type RegionalRehomeCandidateSkip = { + reason: + | 'candidate_stale' + | 'source_ineligible' + | 'source_unclean' + | 'source_control_inactive' + | 'target_unclean' + | 'no_eligible_target' + | 'no_target_headroom' + cellId?: string + sqlFailures?: number + reconnects?: number +} + +function cellUncleanSkip( + reason: 'source_unclean' | 'target_unclean', + cellId: string, + safety: SqlRow | undefined +): RegionalRehomeCandidateSkip { + return { + reason, + cellId, + sqlFailures: safety === undefined ? undefined : integer(safety, 'sql_failures'), + reconnects: safety === undefined ? undefined : integer(safety, 'reconnects') + } +} + +// Candidate skips are otherwise invisible: they neither latch the control off +// nor produce attempts, so an operator cannot tell "skipping" from "idle". +// Cell ids and counters only — never free-form error text. +function aggregateRegionalRehomeCandidateSkips( + skips: readonly RegionalRehomeCandidateSkip[] +): Record { + // `candidates` counts skipped candidate iterations, not distinct cells: one + // unclean cell blocking six candidates reports candidates=6 on one cellId. + const aggregated = new Map() + for (const skip of skips) { + const key = `${skip.reason}:${skip.cellId ?? ''}` + const entry = aggregated.get(key) + if (entry) entry.candidates += 1 + else aggregated.set(key, { ...skip, candidates: 1 }) + } + return { + event: 'orca_relay_regional_rehome_candidates_skipped', + skips: [...aggregated.values()] + } +} + +function regionalRehomeCellSafetyIsClean( + safety: SqlRow | undefined, + runtime: SqlRow, + now: number +): boolean { + return ( + safety !== undefined && + text(safety, 'cell_incarnation') === text(runtime, 'cell_incarnation') && + integer(safety, 'observed_at') > now - 60_000 && + integer(safety, 'sql_failures') <= REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT && + integer(safety, 'reconnects') <= REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT && + integer(safety, 'control_activity_recovery_failures') === 0 && + !regionalRehomePoolPressure({ + databasePoolWaitersMax: integer(safety, 'database_pool_waiters_max'), + databasePoolWaitMsMax: integer(safety, 'database_pool_wait_ms_max') + }) + ) +} + +function assertMigrationPair(row: SqlRow, sourceCellId: string, targetCellId: string): void { + if ( + text(row, 'source_cell_id') !== sourceCellId || + text(row, 'target_cell_id') !== targetCellId + ) { + throw new Error('migration_pair_mismatch') + } +} + +function matchesExactCellSet( + actual: ReadonlySet, + expected: readonly string[] +): boolean { + const expectedSet = new Set(expected) + return ( + expectedSet.size === expected.length && + actual.size === expectedSet.size && + [...actual].every((cellId) => expectedSet.has(cellId)) + ) +} + +function assertCurrentMigrationAssignment( + assignment: SqlRow | undefined, + migrationRow: SqlRow +): asserts assignment is SqlRow { + if ( + !assignment || + text(assignment, 'cell_id') !== text(migrationRow, 'target_cell_id') || + integer(assignment, 'assignment_epoch') !== integer(migrationRow, 'assignment_epoch') + ) { + throw new Error('migration_assignment_mismatch') + } +} + +function assertMigrationRecoveryMetadata( + migrationRow: SqlRow, + incarnation: SqlRow | undefined, + pin: SqlRow | undefined +): void { + if (pin && !incarnation) { + throw new Error('drain_migration_source_incarnation_mismatch') + } + if ( + pin && + incarnation && + (text(pin, 'source_cell_id') !== text(migrationRow, 'source_cell_id') || + text(pin, 'target_cell_id') !== text(migrationRow, 'target_cell_id') || + text(pin, 'source_cell_incarnation') !== + text(incarnation, 'source_cell_incarnation') || + text(pin, 'target_cell_incarnation') !== + text(incarnation, 'target_cell_incarnation') || + integer(pin, 'source_request_units') !== + integer(migrationRow, 'source_request_units') || + integer(pin, 'target_reserved_units') !== + integer(migrationRow, 'target_reserved_units')) + ) { + throw new Error('migration_activity_topology_mismatch') + } +} + +function assertAssignmentActivityAccounting( + assignment: SqlRow, + leases: SqlRow[], + migrationRow: SqlRow +): void { + if ( + integer(migrationRow, 'target_reserved_units') !== + integer(migrationRow, 'source_request_units') + 1 + ) { + throw new Error('migration_activity_lease_shape_mismatch') + } + const counts = emptyActivityCounts() + for (const lease of leases) { + const kind = activityKind(lease) + const requestUnits = integer(lease, 'request_units') + if (kind === 'migration') { + if ( + text(lease, 'activity_id') !== + migrationActivityId(integer(migrationRow, 'assignment_epoch')) || + text(lease, 'cell_id') !== text(migrationRow, 'target_cell_id') || + requestUnits !== integer(migrationRow, 'source_request_units') + ) { + throw new Error('migration_activity_lease_shape_mismatch') + } + } else if (requestUnits !== ACTIVITY_REQUEST_UNITS[kind]) { + throw new Error('migration_activity_lease_shape_mismatch') + } + counts[kind]++ + } + assertAssignmentActivityCounts(assignment, leases, 1) +} + +function assertAssignmentActivityCounts( + assignment: SqlRow, + leases: SqlRow[], + expectedMigrations: number +): void { + const counts = activityCounts(leases) + if ( + (Object.keys(ACTIVITY_COLUMN) as AssignmentActivityKind[]).some( + (kind) => integer(assignment, ACTIVITY_COLUMN[kind]) !== counts[kind] + ) || + counts.migration !== expectedMigrations + ) { + throw new Error('migration_activity_accounting_mismatch') + } +} + +async function assertCellReservationAccounting( + transaction: RelayDatabase, + cells: SqlRow[], + cellIds: string[] +): Promise { + const uniqueCellIds = [...new Set(cellIds)] + const rows = await transaction.query( + `SELECT cell_id, COALESCE(SUM(request_units), 0) AS request_units + FROM relay_assignment_activity_leases + WHERE cell_id IN (${uniqueCellIds.map(() => '?').join(', ')}) + GROUP BY cell_id`, + uniqueCellIds + ) + const unitsByCell = new Map( + rows.map((row) => [text(row, 'cell_id'), integer(row, 'request_units')]) + ) + for (const cellId of uniqueCellIds) { + const cell = cells.find((row) => text(row, 'cell_id') === cellId) + if (!cell || integer(cell, 'reserved_requests') !== (unitsByCell.get(cellId) ?? 0)) { + throw new Error('migration_cell_reservation_accounting_mismatch') + } + } +} + +function deadSourceCompletionResult( + row: SqlRow, + changed: boolean +): DeadSourceCompletionResult { + return { + changed, + assignmentEpoch: integer(row, 'assignment_epoch'), + sourceCellId: text(row, 'source_cell_id'), + targetCellId: text(row, 'target_cell_id') + } +} + +function cellDrainAttempt(row: SqlRow): CellDrainAttempt { + const state = text(row, 'state') + if ( + ![ + 'prepared', + 'send-may-have-started', + 'application-receipt', + 'proven-not-delivered' + ].includes(state) + ) { + throw new Error('invalid_drain_attempt_state') + } + return { + attemptId: text(row, 'attempt_id'), + cellId: text(row, 'cell_id'), + cellIncarnation: text(row, 'cell_incarnation'), + traceValue: text(row, 'trace_value'), + plannedGraceMs: integer(row, 'planned_grace_ms'), + state: state as CellDrainAttemptState, + preparedAt: integer(row, 'prepared_at'), + sendMayHaveStartedAt: optionalInteger(row, 'send_may_have_started_at'), + sendPermitExpiresAt: optionalInteger(row, 'send_permit_expires_at'), + applicationReceiptAt: optionalInteger(row, 'application_receipt_at'), + backendSuccessStatus: optionalInteger(row, 'backend_success_status'), + backendInstance: optionalText(row, 'backend_instance'), + receiptCellIncarnation: optionalText(row, 'receipt_cell_incarnation'), + retryAfter: optionalInteger(row, 'retry_after'), + recoverForwardAttemptedAt: optionalInteger(row, 'recover_forward_attempted_at'), + provenNotDeliveredAt: optionalInteger(row, 'proven_not_delivered_at') + } +} + +function optionalInteger(row: SqlRow, field: string): number | undefined { + if (row[field] === null || row[field] === undefined) return undefined + return integer(row, field) +} + +function optionalText(row: SqlRow, field: string): string | undefined { + if (row[field] === null || row[field] === undefined) return undefined + return text(row, field) +} + +function migrationHasExactActiveTarget(row: SqlRow, targetCellId: string): boolean { + return ( + integer(row, 'target_control_active') === 1 && + optionalText(row, 'current_cell_id') === targetCellId && + optionalInteger(row, 'current_assignment_epoch') === integer(row, 'assignment_epoch') + ) +} + +function assignmentKey(userId: string, relayHostId: string): string { + return `${userId}\u0000${relayHostId}` +} + +function emptyActivityCounts(): Record { + return { + control: 0, + splice: 0, + invite: 0, + install: 0, + confirmation: 0, + migration: 0 + } +} diff --git a/cloud/apps/relay/src/cell-admission-migration-registration.ts b/cloud/apps/relay/src/cell-admission-migration-registration.ts new file mode 100644 index 00000000000..a6cd356a42b --- /dev/null +++ b/cloud/apps/relay/src/cell-admission-migration-registration.ts @@ -0,0 +1,346 @@ +import { + isRelayCellConnectionHardCap, + RELAY_DEFAULT_REGION, + RELAY_REGIONS, + relayCellAdmissionBounds, + type RelayCellConnectionHardCap, + type RelayRegion +} from '@orca-cloud/relay-contract' +import { + decodeMembership, + encodeMembership, + lockedSelector, + normalizeMembership, + requireSelectorMatchesAdmission, + type CellAdmissionMembership, + type CellAdmissionSelector +} from './cell-admission-selector.js' +import type { RelayDatabase, SqlRow } from './database.js' + +export type MigrationCellRegistration = { + id: string + url: string + capacityRequests: number + connectionHardCap: RelayCellConnectionHardCap + connectionUnobservedBound: number + region?: RelayRegion +} + +type NormalizedMigrationCellRegistration = MigrationCellRegistration & { region: RelayRegion } + +type AddMigrationCellsInput = { + attemptId: string + expectedGeneration: number + cells: MigrationCellRegistration[] +} + +const SELECTOR_ID = 'general' + +export class RelayMigrationCellRegistrar { + constructor( + private readonly database: RelayDatabase, + private readonly now: () => number + ) {} + + async add(input: AddMigrationCellsInput): Promise<{ + changed: boolean + selector: CellAdmissionSelector + }> { + validateAttempt(input) + const cells = normalizeMigrationCells(input.cells) + const encodedCells = encodeCells(cells) + return await this.database.transaction(async (transaction) => { + const inventory = await transaction.queryLocked( + `SELECT * FROM relay_cells ORDER BY cell_id ASC` + ) + const selector = await lockedSelector(transaction) + await requireSelectorMatchesAdmission(transaction, selector) + const intent = ( + await transaction.queryLocked( + `SELECT * FROM relay_admission_selector_intents WHERE attempt_id = ?`, + [input.attemptId] + ) + )[0] + const addition = ( + await transaction.queryLocked( + `SELECT * FROM relay_admission_selector_cell_additions WHERE attempt_id = ?`, + [input.attemptId] + ) + )[0] + if (Boolean(intent) !== Boolean(addition)) { + throw new Error('admission_selector_attempt_mismatch') + } + if (intent && addition) { + const membership = decodeMembership(text(intent, 'membership_json')) + if ( + integer(intent, 'expected_generation') !== input.expectedGeneration || + integer(intent, 'intended_generation') !== input.expectedGeneration + 1 || + text(addition, 'cells_json') !== encodedCells || + encodeMembership(membership) !== + encodeMembership( + membershipAfterAddition( + decodeMembership(text(intent, 'previous_membership_json')), + cells + ) + ) + ) { + throw new Error('admission_selector_attempt_mismatch') + } + if ( + selector.generation === input.expectedGeneration + 1 && + selector.attemptId === input.attemptId && + encodeMembership(selector.membership) === encodeMembership(membership) + ) { + await requireExactAddedCells(transaction, inventory, cells) + return { changed: false, selector } + } + throw new Error('admission_selector_generation_mismatch') + } + if (selector.generation < 1) { + throw new Error('admission_selector_boundary_inactive') + } + if (selector.generation !== input.expectedGeneration) { + throw new Error('admission_selector_generation_mismatch') + } + const inventoryIds = new Set(inventory.map((row) => text(row, 'cell_id'))) + const inventoryUrls = new Set(inventory.map((row) => text(row, 'cell_url'))) + if (cells.some((cell) => inventoryIds.has(cell.id) || inventoryUrls.has(cell.url))) { + throw new Error('admission_selector_cell_already_exists') + } + return await this.commit(transaction, input, cells, selector, encodedCells) + }) + } + + private async commit( + transaction: RelayDatabase, + input: AddMigrationCellsInput, + cells: NormalizedMigrationCellRegistration[], + selector: CellAdmissionSelector, + encodedCells: string + ): Promise<{ changed: true; selector: CellAdmissionSelector }> { + const membership = membershipAfterAddition(selector.membership, cells) + const encodedMembership = encodeMembership(membership) + const now = this.now() + await transaction.query( + `INSERT INTO relay_admission_selector_intents + (attempt_id, expected_generation, intended_generation, previous_membership_json, + membership_json, created_at, committed_at) + VALUES (?, ?, ?, ?, ?, ?, NULL)`, + [ + input.attemptId, + input.expectedGeneration, + input.expectedGeneration + 1, + encodeMembership(selector.membership), + encodedMembership, + now + ] + ) + await transaction.query( + `INSERT INTO relay_admission_selector_cell_additions (attempt_id, cells_json) + VALUES (?, ?)`, + [input.attemptId, encodedCells] + ) + for (const cell of cells) await insertCell(transaction, cell, now) + const intendedGeneration = input.expectedGeneration + 1 + const updated = await transaction.query( + `UPDATE relay_admission_selectors + SET generation = ?, attempt_id = ?, membership_json = ?, updated_at = ? + WHERE selector_id = ? AND generation = ?`, + [ + intendedGeneration, + input.attemptId, + encodedMembership, + now, + SELECTOR_ID, + input.expectedGeneration + ] + ) + if (integer(updated[0]!, 'changes') !== 1) { + throw new Error('admission_selector_generation_mismatch') + } + await transaction.query( + `UPDATE relay_admission_selector_intents SET committed_at = ? + WHERE attempt_id = ?`, + [now, input.attemptId] + ) + return { + changed: true, + selector: { + generation: intendedGeneration, + attemptId: input.attemptId, + membership + } + } + } +} + +function encodeCells(cells: NormalizedMigrationCellRegistration[]): string { + return JSON.stringify( + cells.map((cell) => + cell.region === RELAY_DEFAULT_REGION + ? { + id: cell.id, + url: cell.url, + capacityRequests: cell.capacityRequests, + connectionHardCap: cell.connectionHardCap, + connectionUnobservedBound: cell.connectionUnobservedBound + } + : cell + ) + ) +} + +async function insertCell( + transaction: RelayDatabase, + cell: NormalizedMigrationCellRegistration, + now: number +): Promise { + await transaction.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES (?, ?, 1, ?, 0, 0, ?, ?)`, + [cell.id, cell.url, cell.capacityRequests, now, now] + ) + await transaction.query( + `INSERT INTO relay_cell_regions (cell_id, region) VALUES (?, ?)`, + [cell.id, cell.region] + ) + await transaction.query( + `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at) + VALUES (?, 'migration-only', ?)`, + [cell.id, now] + ) + await transaction.query( + `INSERT INTO relay_cell_connection_limits + (cell_id, hard_cap, unobserved_bound, updated_at) + VALUES (?, ?, ?, ?)`, + [cell.id, cell.connectionHardCap, cell.connectionUnobservedBound, now] + ) +} + +function normalizeMigrationCells( + input: MigrationCellRegistration[] +): NormalizedMigrationCellRegistration[] { + if (input.length === 0 || input.length > 128) { + throw new Error('invalid_admission_selector_cells') + } + const cells = input + .map((cell) => ({ ...cell, region: cell.region ?? RELAY_DEFAULT_REGION })) + .sort((left, right) => left.id.localeCompare(right.id)) + if ( + new Set(cells.map(({ id }) => id)).size !== cells.length || + new Set(cells.map(({ url }) => url)).size !== cells.length + ) { + throw new Error('admission_selector_duplicate_cell') + } + for (const cell of cells) validateMigrationCell(cell) + return cells +} + +function validateMigrationCell(cell: NormalizedMigrationCellRegistration): void { + if ( + cell.id.length === 0 || + cell.id.length > 128 || + cell.url.length === 0 || + cell.url.length > 2_048 || + !Number.isSafeInteger(cell.capacityRequests) || + cell.capacityRequests <= 0 || + cell.capacityRequests > 100_000 || + !isRelayCellConnectionHardCap(cell.connectionHardCap) || + !Number.isSafeInteger(cell.connectionUnobservedBound) || + cell.connectionUnobservedBound < 0 || + cell.connectionUnobservedBound > + relayCellAdmissionBounds(cell.connectionHardCap).maxUnobservedBound || + !RELAY_REGIONS.includes(cell.region) + ) { + throw new Error('invalid_admission_selector_cell') + } +} + +function membershipAfterAddition( + membership: CellAdmissionMembership, + cells: NormalizedMigrationCellRegistration[] +): CellAdmissionMembership { + return normalizeMembership({ + existingOnly: membership.existingOnly, + migrationOnly: [...membership.migrationOnly, ...cells.map(({ id }) => id)], + general: membership.general + }) +} + +async function requireExactAddedCells( + database: RelayDatabase, + inventory: SqlRow[], + cells: NormalizedMigrationCellRegistration[] +): Promise { + const byId = new Map(inventory.map((row) => [text(row, 'cell_id'), row])) + for (const cell of cells) { + const row = byId.get(cell.id) + const admission = await admissionState(database, cell.id) + const region = await cellRegion(database, cell.id) + const limit = await connectionLimit(database, cell.id) + if ( + !row || + text(row, 'cell_url') !== cell.url || + integer(row, 'enabled') !== 1 || + integer(row, 'capacity_requests') !== cell.capacityRequests || + region !== cell.region || + admission !== 'migration-only' || + !limit || + integer(limit, 'hard_cap') !== cell.connectionHardCap || + integer(limit, 'unobserved_bound') !== cell.connectionUnobservedBound + ) { + throw new Error('admission_selector_attempt_mismatch') + } + } +} + +async function cellRegion(database: RelayDatabase, cellId: string): Promise { + return ( + await database.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [cellId]) + )[0]?.region +} + +async function admissionState(database: RelayDatabase, cellId: string): Promise { + return ( + await database.query( + `SELECT admission_state FROM relay_cell_admission WHERE cell_id = ?`, + [cellId] + ) + )[0]?.admission_state +} + +async function connectionLimit( + database: RelayDatabase, + cellId: string +): Promise { + return ( + await database.query( + `SELECT hard_cap, unobserved_bound FROM relay_cell_connection_limits + WHERE cell_id = ?`, + [cellId] + ) + )[0] +} + +function validateAttempt(input: AddMigrationCellsInput): void { + if (!/^[A-Za-z0-9_-]{8,128}$/.test(input.attemptId)) { + throw new Error('invalid_admission_selector_attempt') + } + if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) { + throw new Error('invalid_admission_selector_generation') + } +} + +function integer(row: SqlRow, field: string): number { + const value = Number(row[field]) + if (!Number.isSafeInteger(value)) throw new Error(`invalid integer field ${field}`) + return value +} + +function text(row: SqlRow, field: string): string { + const value = row[field] + if (typeof value !== 'string') throw new Error(`invalid text field ${field}`) + return value +} diff --git a/cloud/apps/relay/src/cell-admission-selector.test.ts b/cloud/apps/relay/src/cell-admission-selector.test.ts new file mode 100644 index 00000000000..41b84f91d79 --- /dev/null +++ b/cloud/apps/relay/src/cell-admission-selector.test.ts @@ -0,0 +1,580 @@ +import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract' +import { createHash } from 'node:crypto' +import { afterEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { encodeMembership } from './cell-admission-selector.js' +import { + openInMemoryRelayDatabase, + type RelayDatabase, + type RelayLockOptions, + type SqlRow +} from './database.js' + +const CELLS = [ + { id: 'c1', url: 'https://c1.example.com', capacityRequests: 20 }, + { id: 'c2', url: 'https://c2.example.com', capacityRequests: 20 }, + { id: 'c3', url: 'https://c3.example.com', capacityRequests: 20 } +] + +const INITIAL_MEMBERSHIP = { + existingOnly: ['c1'], + migrationOnly: ['c2'], + general: ['c3'] +} +const BASE_MEMBERSHIP = { + existingOnly: [], + migrationOnly: [], + general: ['c1', 'c2', 'c3'] +} + +class FailAfterIntentDatabase implements RelayDatabase { + private transactionsUntilFailure = Number.POSITIVE_INFINITY + + constructor(private readonly delegate: RelayDatabase) {} + + failSecondTransaction(): void { + this.transactionsUntilFailure = 2 + } + + async query(sql: string, params?: unknown[]): Promise { + return await this.delegate.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + return await this.delegate.queryLocked(sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + this.transactionsUntilFailure-- + if (this.transactionsUntilFailure === 0) { + this.transactionsUntilFailure = Number.POSITIVE_INFINITY + throw new Error('injected_commit_ambiguity') + } + return await this.delegate.transaction(operation) + } + + async close(): Promise { + await this.delegate.close() + } +} + +class AfterTransactionDatabase implements RelayDatabase { + private afterTransaction: (() => Promise) | undefined + + constructor(private readonly delegate: RelayDatabase) {} + + runAfterNextTransaction(operation: () => Promise): void { + this.afterTransaction = operation + } + + async query(sql: string, params?: unknown[]): Promise { + return await this.delegate.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + return await this.delegate.queryLocked(sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + const result = await this.delegate.transaction(operation) + const after = this.afterTransaction + this.afterTransaction = undefined + if (after) await after() + return result + } + + async close(): Promise { + await this.delegate.close() + } +} + +function membershipSha256(membership: typeof INITIAL_MEMBERSHIP): string { + return createHash('sha256').update(encodeMembership(membership)).digest('hex') +} + +const BASE_MEMBERSHIP_SHA256 = membershipSha256(BASE_MEMBERSHIP) + +describe('relay cell admission selector', () => { + let database: RelayDatabase | undefined + + afterEach(async () => await database?.close()) + + async function setup( + now: () => number, + requireLiveCells = false + ): Promise { + database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, now, { + requireLiveCells, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells(CELLS) + return store + } + + async function heartbeat(store: RelayAssignmentStore, cellIndex: number): Promise { + const cell = CELLS[cellIndex]! + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: `${cellIndex + 1}1111111-1111-4111-8111-111111111111`, + startedAt: 50, + ready: true, + observedRequests: 0 + }) + } + + it('preserves sticky assignments while reserving ordinary placement for general cells', async () => { + const store = await setup(() => 100) + const existing = { userId: 'existing', relayHostId: 'host000000000001' } + expect(await store.assign(existing)).toMatchObject({ cellId: 'c1' }) + + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_001', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + + expect(await store.assign(existing)).toMatchObject({ cellId: 'c1' }) + await expect( + store.assign({ userId: 'new', relayHostId: 'host000000000002' }) + ).resolves.toMatchObject({ cellId: 'c3' }) + await expect(store.startEvacuation(existing, 'c2')).resolves.toMatchObject({ + sourceCellId: 'c1', + targetCellId: 'c2' + }) + }) + + it('excludes existing-only and migration-only cells from dormant placement', async () => { + let now = 100 + const store = await setup(() => now) + const identity = { userId: 'dormant', relayHostId: 'host000000000001' } + const assignment = await store.assign(identity) + const control = await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.releaseActivity(identity, control) + now += ASSIGNMENT_LIMITS.dormantTtlMs + 1 + + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_002', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + + await expect(store.rebalanceDormant(identity, 'c2')).rejects.toThrow( + 'target_cell_unavailable' + ) + await expect(store.rebalanceDormant(identity, 'c3')).resolves.toMatchObject({ + cellId: 'c3' + }) + }) + + it('does not recover an unfenced dead existing-only assignment', async () => { + let now = 100 + const store = await setup(() => now, true) + await heartbeat(store, 0) + await heartbeat(store, 1) + await heartbeat(store, 2) + const identity = { userId: 'dead-source', relayHostId: 'host000000000001' } + expect(await store.assign(identity)).toMatchObject({ cellId: 'c1' }) + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_003', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + + now += 45_001 + await heartbeat(store, 1) + await heartbeat(store, 2) + expect(await store.evacuateDeadCells()).toBe(0) + expect(await store.resolve(identity)).toBeNull() + }) + + it('commits atomically and resolves an ambiguous response idempotently', async () => { + const store = await setup(() => 100) + const input = { + attemptId: 'cutover_004', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + } + + await expect(store.applyCellAdmissionSelector(input)).resolves.toMatchObject({ + changed: true, + selector: { generation: 1, membership: INITIAL_MEMBERSHIP } + }) + await expect(store.applyCellAdmissionSelector(input)).resolves.toMatchObject({ + changed: false, + selector: { generation: 1, membership: INITIAL_MEMBERSHIP } + }) + await expect(store.inspectCellAdmissionSelector(input.attemptId)).resolves.toMatchObject({ + intent: { state: 'committed' } + }) + expect(await store.cellDeploymentStatus('c1')).toMatchObject({ + enabled: false, + admissionState: 'existing-only' + }) + expect(await store.cellDeploymentStatus('c2')).toMatchObject({ + enabled: true, + admissionState: 'migration-only' + }) + }) + + it('rejects a generation-zero membership change between intent and commit', async () => { + const delegate = await openInMemoryRelayDatabase() + const hooked = new AfterTransactionDatabase(delegate) + database = hooked + const store = new RelayAssignmentStore(hooked, () => 100) + await store.reconcileCells(CELLS) + const inspected = (await store.inspectCellAdmissionSelector()).selector.membership + const changed = { + existingOnly: ['c2'], + migrationOnly: [], + general: ['c1', 'c3'] + } + hooked.runAfterNextTransaction(async () => { + await delegate.transaction(async (transaction) => { + await transaction.query( + `UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, + ['c2'] + ) + await transaction.query( + `UPDATE relay_cell_admission SET admission_state = ? WHERE cell_id = ?`, + ['existing-only', 'c2'] + ) + await transaction.query( + `UPDATE relay_admission_selectors SET membership_json = ? + WHERE selector_id = ? AND generation = 0`, + [encodeMembership(changed), 'general'] + ) + }) + }) + + await expect(store.applyCellAdmissionSelector({ + attemptId: 'cutover_race_001', + expectedGeneration: 0, + expectedMembershipSha256: membershipSha256(inspected), + membership: inspected + })).rejects.toThrow('admission_selector_membership_mismatch') + await expect(store.applyCellAdmissionSelector({ + attemptId: 'cutover_race_001', + expectedGeneration: 0, + membership: inspected + })).rejects.toThrow('admission_selector_membership_fingerprint_required') + await expect(store.inspectCellAdmissionSelector()).resolves.toMatchObject({ + selector: { generation: 0, membership: changed } + }) + }) + + it('preserves admission age for cells unchanged by a selector CAS', async () => { + let now = 100 + const store = await setup(() => now) + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_age_001', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + + now = 200 + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_age_002', + expectedGeneration: 1, + membership: { + existingOnly: ['c1'], + migrationOnly: [], + general: ['c2', 'c3'] + } + }) + + expect( + await database!.query( + `SELECT cell_id, admission_state, updated_at + FROM relay_cell_admission ORDER BY cell_id` + ) + ).toEqual([ + { cell_id: 'c1', admission_state: 'existing-only', updated_at: 100 }, + { cell_id: 'c2', admission_state: 'general', updated_at: 200 }, + { cell_id: 'c3', admission_state: 'general', updated_at: 100 } + ]) + }) + + it('persists failed CAS intent without mutating current membership', async () => { + const store = await setup(() => 100) + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_005', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + + await expect( + store.applyCellAdmissionSelector({ + attemptId: 'stale_attempt', + expectedGeneration: 5, + membership: { + existingOnly: ['c1'], + migrationOnly: [], + general: ['c2', 'c3'] + } + }) + ).rejects.toThrow('admission_selector_generation_mismatch') + await expect(store.inspectCellAdmissionSelector('stale_attempt')).resolves.toMatchObject({ + selector: { generation: 1, membership: INITIAL_MEMBERSHIP }, + intent: { state: 'diverged' } + }) + }) + + it('rejects duplicate or incomplete selector membership before mutation', async () => { + const store = await setup(() => 100) + + await expect( + store.applyCellAdmissionSelector({ + attemptId: 'duplicate_001', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: { + existingOnly: ['c1', 'c1'], + migrationOnly: ['c2'], + general: ['c3'] + } + }) + ).rejects.toThrow('admission_selector_duplicate_cell') + await expect( + store.applyCellAdmissionSelector({ + attemptId: 'incomplete_001', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: { + existingOnly: ['c1'], + migrationOnly: ['c2'], + general: [] + } + }) + ).rejects.toThrow('admission_selector_incomplete_membership') + await expect(store.inspectCellAdmissionSelector()).resolves.toMatchObject({ + selector: { + generation: 0, + membership: { existingOnly: [], migrationOnly: [], general: ['c1', 'c2', 'c3'] } + } + }) + }) + + it('inspects an unchanged durable intent after an ambiguous apply failure', async () => { + const underlying = await openInMemoryRelayDatabase() + const failingDatabase = new FailAfterIntentDatabase(underlying) + database = failingDatabase + const store = new RelayAssignmentStore(failingDatabase, () => 100) + await store.reconcileCells(CELLS) + failingDatabase.failSecondTransaction() + + const input = { + attemptId: 'ambiguous_001', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + } + await expect(store.applyCellAdmissionSelector(input)).rejects.toThrow( + 'injected_commit_ambiguity' + ) + await expect(store.inspectCellAdmissionSelector(input.attemptId)).resolves.toMatchObject({ + selector: { generation: 0 }, + intent: { state: 'unchanged' } + }) + await expect(store.applyCellAdmissionSelector(input)).resolves.toMatchObject({ + changed: true, + selector: { generation: 1, membership: INITIAL_MEMBERSHIP } + }) + }) + + it('allows only one concurrent CAS and never re-enables legacy cells', async () => { + const store = await setup(() => 100) + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_006', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + await expect(store.setCellEnabled('c1', true)).rejects.toThrow( + 'admission_selector_boundary_active' + ) + + const nextMembership = { + existingOnly: ['c1'], + migrationOnly: [], + general: ['c2', 'c3'] + } + const results = await Promise.allSettled([ + store.applyCellAdmissionSelector({ + attemptId: 'promote_001', + expectedGeneration: 1, + membership: nextMembership + }), + store.applyCellAdmissionSelector({ + attemptId: 'promote_002', + expectedGeneration: 1, + membership: nextMembership + }) + ]) + expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1) + expect(results.filter(({ status }) => status === 'rejected')).toHaveLength(1) + + await expect( + store.applyCellAdmissionSelector({ + attemptId: 'reenable_001', + expectedGeneration: 2, + membership: { + existingOnly: [], + migrationOnly: [], + general: ['c1', 'c2', 'c3'] + } + }) + ).rejects.toThrow('admission_selector_legacy_reenable') + }) + + it('preserves the selector boundary across compatible reconciliation', async () => { + const store = await setup(() => 100) + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_007', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + + await expect(store.reconcileCells(CELLS, false)).resolves.toBeUndefined() + await expect(store.reconcileCells([CELLS[0]!, CELLS[2]!])).rejects.toThrow( + 'admission_selector_boundary_active' + ) + await expect(store.inspectCellAdmissionSelector()).resolves.toMatchObject({ + selector: { generation: 1, membership: INITIAL_MEMBERSHIP } + }) + }) + + it('atomically adds exact migration-only cells after the selector boundary', async () => { + const store = await setup(() => 100) + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_008', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + const input = { + attemptId: 'add_cells_001', + expectedGeneration: 1, + cells: [ + { + id: 'c5', + url: 'https://c5.example.com', + capacityRequests: 4_000, + connectionHardCap: 1_000 as const, + connectionUnobservedBound: 60 + }, + { + id: 'c4', + url: 'https://c4.example.com', + capacityRequests: 4_000, + connectionHardCap: 600 as const, + connectionUnobservedBound: 60 + } + ] + } + + await expect(store.addMigrationCells(input)).resolves.toMatchObject({ + changed: true, + selector: { + generation: 2, + attemptId: input.attemptId, + membership: { + existingOnly: ['c1'], + migrationOnly: ['c2', 'c4', 'c5'], + general: ['c3'] + } + } + }) + await expect(store.addMigrationCells(input)).resolves.toMatchObject({ + changed: false, + selector: { generation: 2 } + }) + await expect(store.inspectCellAdmissionSelector(input.attemptId)).resolves.toMatchObject({ + intent: { state: 'committed' } + }) + await expect(store.cellDeploymentStatus('c4')).resolves.toMatchObject({ + enabled: true, + admissionState: 'migration-only', + cellUrl: 'https://c4.example.com', + capacityRequests: 4_000, + connectionCapacity: { + hardCap: 600, + unobservedBound: 60 + } + }) + await expect(store.cellDeploymentStatus('c5')).resolves.toMatchObject({ + connectionCapacity: { + hardCap: 1_000, + ordinaryConnectionLimit: 900, + normalAdmissionPause: 840 + } + }) + }) + + it('rejects additive cells before cutover and exact-attempt config reuse', async () => { + const store = await setup(() => 100) + const cell = { + id: 'c4', + url: 'https://c4.example.com', + capacityRequests: 4_000, + connectionHardCap: 600 as const, + connectionUnobservedBound: 60 + } + await expect( + store.addMigrationCells({ + attemptId: 'add_cells_002', + expectedGeneration: 0, + cells: [cell] + }) + ).rejects.toThrow('admission_selector_boundary_inactive') + await store.applyCellAdmissionSelector({ + attemptId: 'cutover_009', + expectedGeneration: 0, + expectedMembershipSha256: BASE_MEMBERSHIP_SHA256, + membership: INITIAL_MEMBERSHIP + }) + await store.addMigrationCells({ + attemptId: 'add_cells_002', + expectedGeneration: 1, + cells: [cell] + }) + await expect( + store.addMigrationCells({ + attemptId: 'add_cells_002', + expectedGeneration: 1, + cells: [{ ...cell, capacityRequests: 3_999 }] + }) + ).rejects.toThrow('admission_selector_attempt_mismatch') + await expect( + store.applyCellAdmissionSelector({ + attemptId: 'add_cells_002', + expectedGeneration: 2, + membership: { + existingOnly: ['c1'], + migrationOnly: ['c2', 'c4'], + general: ['c3'] + } + }) + ).rejects.toThrow('admission_selector_attempt_mismatch') + }) +}) diff --git a/cloud/apps/relay/src/cell-admission-selector.ts b/cloud/apps/relay/src/cell-admission-selector.ts new file mode 100644 index 00000000000..92764e53e40 --- /dev/null +++ b/cloud/apps/relay/src/cell-admission-selector.ts @@ -0,0 +1,505 @@ +import { createHash } from 'node:crypto' +import type { RelayDatabase, SqlRow } from './database.js' + +export const CELL_ADMISSION_STATES = [ + 'existing-only', + 'migration-only', + 'general' +] as const + +export type CellAdmissionState = (typeof CELL_ADMISSION_STATES)[number] + +export type CellAdmissionMembership = { + existingOnly: string[] + migrationOnly: string[] + general: string[] +} + +export type CellAdmissionSelector = { + generation: number + attemptId: string | null + membership: CellAdmissionMembership +} + +export type CellAdmissionSelectorInspection = { + selector: CellAdmissionSelector + intent: null | { + attemptId: string + expectedGeneration: number + intendedGeneration: number + previousMembership: CellAdmissionMembership + membership: CellAdmissionMembership + state: 'unchanged' | 'committed' | 'diverged' + } +} + +type ApplySelectorInput = { + attemptId: string + expectedGeneration: number + expectedMembershipSha256?: string + membership: CellAdmissionMembership +} + +const SELECTOR_ID = 'general' + +export function stateFromEnabled(enabled: boolean): CellAdmissionState { + return enabled ? 'general' : 'existing-only' +} + +export function enabledForState(state: CellAdmissionState): number { + return state === 'existing-only' ? 0 : 1 +} + +export function parseCellAdmissionState(value: string): CellAdmissionState { + if (!CELL_ADMISSION_STATES.includes(value as CellAdmissionState)) { + throw new Error('invalid_cell_admission_state') + } + return value as CellAdmissionState +} + +export async function ensureCellAdmission( + transaction: RelayDatabase, + cellId: string, + fallback: CellAdmissionState, + now: number +): Promise { + await transaction.query( + `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at) + VALUES (?, ?, ?) ON CONFLICT (cell_id) DO NOTHING`, + [cellId, fallback, now] + ) +} + +export async function cellAdmissionState( + database: RelayDatabase, + cellId: string +): Promise { + const row = ( + await database.query( + `SELECT admission_state FROM relay_cell_admission WHERE cell_id = ?`, + [cellId] + ) + )[0] + if (!row) throw new Error('cell_admission_missing') + return admissionState(row) +} + +export async function cellAdmissionStates( + database: RelayDatabase +): Promise> { + const rows = await database.query( + `SELECT cell.cell_id, admission.admission_state + FROM relay_cells cell + LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id + ORDER BY cell.cell_id ASC` + ) + return new Map(rows.map((row) => [text(row, 'cell_id'), admissionState(row)])) +} + +export async function setCellAdmissionBeforeBoundary( + transaction: RelayDatabase, + cellId: string, + state: CellAdmissionState, + now: number +): Promise { + const cells = await transaction.queryLocked( + `SELECT cell_id FROM relay_cells ORDER BY cell_id ASC` + ) + if (!cells.some((row) => text(row, 'cell_id') === cellId)) { + throw new Error('cell_not_found') + } + if ((await lockedSelector(transaction)).generation > 0) { + throw new Error('admission_selector_boundary_active') + } + const updated = await transaction.query( + `UPDATE relay_cells + SET updated_at = CASE WHEN enabled <> ? THEN ? ELSE updated_at END, enabled = ? + WHERE cell_id = ?`, + [enabledForState(state), now, enabledForState(state), cellId] + ) + if (integer(updated[0]!, 'changes') !== 1) throw new Error('cell_not_found') + await ensureCellAdmission(transaction, cellId, state, now) + await transaction.query( + `UPDATE relay_cell_admission + SET updated_at = CASE WHEN admission_state <> ? THEN ? ELSE updated_at END, + admission_state = ? + WHERE cell_id = ?`, + [state, now, state, cellId] + ) + await synchronizeCellAdmissionBoundary(transaction, now) +} + +export async function synchronizeCellAdmissionBoundary( + transaction: RelayDatabase, + now: number +): Promise { + const selector = await lockedSelector(transaction) + if (selector.generation > 0) { + await requireSelectorMatchesAdmission(transaction, selector) + return + } + await transaction.query( + `UPDATE relay_admission_selectors SET membership_json = ?, updated_at = ? + WHERE selector_id = ? AND generation = 0`, + [encodeMembership(await currentMembership(transaction)), now, SELECTOR_ID] + ) +} + +export class RelayCellAdmissionSelector { + constructor( + private readonly database: RelayDatabase, + private readonly now: () => number + ) {} + + async apply(input: ApplySelectorInput): Promise<{ + changed: boolean + selector: CellAdmissionSelector + }> { + const membership = normalizeMembership(input.membership) + validateAttempt(input) + const encoded = encodeMembership(membership) + await this.persistIntent(input, membership, encoded) + return await this.database.transaction(async (transaction) => { + const cells = await transaction.queryLocked( + `SELECT cell_id FROM relay_cells ORDER BY cell_id ASC` + ) + requireExactMembership(cells, membership) + const selector = await lockedSelector(transaction) + if ( + selector.generation === input.expectedGeneration + 1 && + selector.attemptId === input.attemptId && + encodeMembership(selector.membership) === encoded + ) { + return { changed: false, selector } + } + if (selector.generation !== input.expectedGeneration) { + throw new Error('admission_selector_generation_mismatch') + } + requireExpectedMembership(selector, input.expectedMembershipSha256) + await requireSelectorMatchesAdmission(transaction, selector) + if (selector.generation > 0) { + const nextNonExisting = new Set([...membership.migrationOnly, ...membership.general]) + if (selector.membership.existingOnly.some((cellId) => nextNonExisting.has(cellId))) { + throw new Error('admission_selector_legacy_reenable') + } + } + const now = this.now() + for (const [state, cellIds] of membershipEntries(membership)) { + for (const cellId of cellIds) { + await transaction.query( + `UPDATE relay_cell_admission + SET updated_at = CASE WHEN admission_state <> ? THEN ? ELSE updated_at END, + admission_state = ? + WHERE cell_id = ?`, + [state, now, state, cellId] + ) + await transaction.query( + `UPDATE relay_cells + SET updated_at = CASE WHEN enabled <> ? THEN ? ELSE updated_at END, enabled = ? + WHERE cell_id = ?`, + [enabledForState(state), now, enabledForState(state), cellId] + ) + } + } + const intendedGeneration = input.expectedGeneration + 1 + const updated = await transaction.query( + `UPDATE relay_admission_selectors + SET generation = ?, attempt_id = ?, membership_json = ?, updated_at = ? + WHERE selector_id = ? AND generation = ?`, + [ + intendedGeneration, + input.attemptId, + encoded, + now, + SELECTOR_ID, + input.expectedGeneration + ] + ) + if (integer(updated[0]!, 'changes') !== 1) { + throw new Error('admission_selector_generation_mismatch') + } + await transaction.query( + `UPDATE relay_admission_selector_intents SET committed_at = ? + WHERE attempt_id = ?`, + [now, input.attemptId] + ) + return { + changed: true, + selector: { + generation: intendedGeneration, + attemptId: input.attemptId, + membership + } + } + }) + } + + async inspect(attemptId?: string): Promise { + return await this.database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`) + const selector = await lockedSelector(transaction) + await requireSelectorMatchesAdmission(transaction, selector) + if (!attemptId) return { selector, intent: null } + const row = ( + await transaction.queryLocked( + `SELECT * FROM relay_admission_selector_intents WHERE attempt_id = ?`, + [attemptId] + ) + )[0] + if (!row) return { selector, intent: null } + const membership = decodeMembership(text(row, 'membership_json')) + const previousMembership = decodeMembership(text(row, 'previous_membership_json')) + const intendedGeneration = integer(row, 'intended_generation') + const committed = + selector.generation === intendedGeneration && + selector.attemptId === attemptId && + encodeMembership(selector.membership) === encodeMembership(membership) + const unchanged = + selector.generation === integer(row, 'expected_generation') && + encodeMembership(selector.membership) === text(row, 'previous_membership_json') + return { + selector, + intent: { + attemptId, + expectedGeneration: integer(row, 'expected_generation'), + intendedGeneration, + previousMembership, + membership, + state: committed ? 'committed' : unchanged ? 'unchanged' : 'diverged' + } + } + }) + } + + private async persistIntent( + input: ApplySelectorInput, + membership: CellAdmissionMembership, + encoded: string + ): Promise { + await this.database.transaction(async (transaction) => { + const cells = await transaction.queryLocked( + `SELECT cell_id FROM relay_cells ORDER BY cell_id ASC` + ) + requireExactMembership(cells, membership) + const selector = await lockedSelector(transaction) + const existing = ( + await transaction.queryLocked( + `SELECT * FROM relay_admission_selector_intents WHERE attempt_id = ?`, + [input.attemptId] + ) + )[0] + if (existing) { + const addition = ( + await transaction.queryLocked( + `SELECT attempt_id FROM relay_admission_selector_cell_additions + WHERE attempt_id = ?`, + [input.attemptId] + ) + )[0] + if ( + addition || + integer(existing, 'expected_generation') !== input.expectedGeneration || + integer(existing, 'intended_generation') !== input.expectedGeneration + 1 || + text(existing, 'membership_json') !== encoded || + (input.expectedMembershipSha256 !== undefined && + membershipSha256(text(existing, 'previous_membership_json')) !== + input.expectedMembershipSha256) + ) { + throw new Error('admission_selector_attempt_mismatch') + } + return + } + requireExpectedMembership(selector, input.expectedMembershipSha256) + await transaction.query( + `INSERT INTO relay_admission_selector_intents + (attempt_id, expected_generation, intended_generation, previous_membership_json, + membership_json, created_at, committed_at) + VALUES (?, ?, ?, ?, ?, ?, NULL)`, + [ + input.attemptId, + input.expectedGeneration, + input.expectedGeneration + 1, + encodeMembership(selector.membership), + encoded, + this.now() + ] + ) + }) + } +} + +function requireExpectedMembership( + selector: CellAdmissionSelector, + expectedSha256: string | undefined +): void { + if (expectedSha256 === undefined) return + if ( + !/^[a-f0-9]{64}$/.test(expectedSha256) || + membershipSha256(encodeMembership(selector.membership)) !== expectedSha256 + ) { + throw new Error('admission_selector_membership_mismatch') + } +} + +function membershipSha256(encoded: string): string { + return createHash('sha256').update(encoded).digest('hex') +} + +export async function lockedSelector( + transaction: RelayDatabase +): Promise { + let row = ( + await transaction.queryLocked( + `SELECT * FROM relay_admission_selectors WHERE selector_id = ?`, + [SELECTOR_ID] + ) + )[0] + if (!row) { + const membership = await currentMembership(transaction) + await transaction.query( + `INSERT INTO relay_admission_selectors + (selector_id, generation, attempt_id, membership_json, updated_at) + VALUES (?, 0, NULL, ?, 0) ON CONFLICT (selector_id) DO NOTHING`, + [SELECTOR_ID, encodeMembership(membership)] + ) + row = ( + await transaction.queryLocked( + `SELECT * FROM relay_admission_selectors WHERE selector_id = ?`, + [SELECTOR_ID] + ) + )[0] + } + if (!row) throw new Error('admission_selector_missing') + return { + generation: integer(row, 'generation'), + attemptId: optionalText(row, 'attempt_id'), + membership: decodeMembership(text(row, 'membership_json')) + } +} + +async function currentMembership(database: RelayDatabase): Promise { + const rows = await database.query( + `SELECT cell.cell_id, admission.admission_state + FROM relay_cells cell + LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id + ORDER BY cell.cell_id ASC` + ) + const membership: CellAdmissionMembership = { + existingOnly: [], + migrationOnly: [], + general: [] + } + for (const row of rows) membership[keyForState(admissionState(row))].push(text(row, 'cell_id')) + return membership +} + +export async function requireSelectorMatchesAdmission( + database: RelayDatabase, + selector: CellAdmissionSelector +): Promise { + const current = await currentMembership(database) + if (encodeMembership(current) !== encodeMembership(selector.membership)) { + throw new Error('admission_selector_membership_drift') + } +} + +export function normalizeMembership( + input: CellAdmissionMembership +): CellAdmissionMembership { + const membership = { + existingOnly: [...input.existingOnly].sort(), + migrationOnly: [...input.migrationOnly].sort(), + general: [...input.general].sort() + } + const all = [...membership.existingOnly, ...membership.migrationOnly, ...membership.general] + if (new Set(all).size !== all.length) throw new Error('admission_selector_duplicate_cell') + return membership +} + +function requireExactMembership(rows: SqlRow[], membership: CellAdmissionMembership): void { + const expected = rows.map((row) => text(row, 'cell_id')).sort() + const actual = [ + ...membership.existingOnly, + ...membership.migrationOnly, + ...membership.general + ].sort() + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error('admission_selector_incomplete_membership') + } +} + +function membershipEntries( + membership: CellAdmissionMembership +): Array<[CellAdmissionState, string[]]> { + return [ + ['existing-only', membership.existingOnly], + ['migration-only', membership.migrationOnly], + ['general', membership.general] + ] +} + +export function encodeMembership(membership: CellAdmissionMembership): string { + return JSON.stringify(normalizeMembership(membership)) +} + +export function decodeMembership(value: string): CellAdmissionMembership { + const parsed = JSON.parse(value) as Partial + if ( + !Array.isArray(parsed.existingOnly) || + !Array.isArray(parsed.migrationOnly) || + !Array.isArray(parsed.general) || + [...parsed.existingOnly, ...parsed.migrationOnly, ...parsed.general].some( + (cellId) => typeof cellId !== 'string' + ) + ) { + throw new Error('admission_selector_invalid_membership') + } + return normalizeMembership({ + existingOnly: parsed.existingOnly as string[], + migrationOnly: parsed.migrationOnly as string[], + general: parsed.general as string[] + }) +} + +function admissionState(row: SqlRow): CellAdmissionState { + return parseCellAdmissionState(text(row, 'admission_state')) +} + +function keyForState(state: CellAdmissionState): keyof CellAdmissionMembership { + if (state === 'existing-only') return 'existingOnly' + if (state === 'migration-only') return 'migrationOnly' + return 'general' +} + +function validateAttempt(input: { + attemptId: string + expectedGeneration: number + expectedMembershipSha256?: string +}): void { + if (!/^[A-Za-z0-9_-]{8,128}$/.test(input.attemptId)) { + throw new Error('invalid_admission_selector_attempt') + } + if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) { + throw new Error('invalid_admission_selector_generation') + } + if (input.expectedGeneration === 0 && input.expectedMembershipSha256 === undefined) { + throw new Error('admission_selector_membership_fingerprint_required') + } +} + +function optionalText(row: SqlRow, field: string): string | null { + if (row[field] === null || row[field] === undefined) return null + return text(row, field) +} + +function integer(row: SqlRow, field: string): number { + const value = Number(row[field]) + if (!Number.isSafeInteger(value)) throw new Error(`invalid integer field ${field}`) + return value +} + +function text(row: SqlRow, field: string): string { + const value = row[field] + if (typeof value !== 'string') throw new Error(`invalid text field ${field}`) + return value +} diff --git a/cloud/apps/relay/src/cell-admission-startup-postgres.test.ts b/cloud/apps/relay/src/cell-admission-startup-postgres.test.ts new file mode 100644 index 00000000000..6f93fecce6f --- /dev/null +++ b/cloud/apps/relay/src/cell-admission-startup-postgres.test.ts @@ -0,0 +1,83 @@ +import pg from 'pg' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { reconcileCellAdmissionAtStartup } from './cell-admission-startup.js' +import type { RelayCellConfig } from './config.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const schema = 'relay_startup_retry_test' +const cell: RelayCellConfig = { + id: 'startup-retry-cell', + url: 'https://startup-retry.example.test', + capacityRequests: 4_000 +} + +afterEach(() => vi.restoreAllMocks()) + +describePostgres('PostgreSQL director startup reconciliation', () => { + const databases: RelayDatabase[] = [] + let scopedDatabaseUrl = '' + + beforeAll(async () => { + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + await client.query(`CREATE SCHEMA ${schema}`) + } finally { + await client.end() + } + const url = new URL(databaseUrl!) + url.searchParams.set('options', `-c search_path=${schema}`) + scopedDatabaseUrl = url.toString() + databases.push( + await openRelayDatabase({ databaseUrl: scopedDatabaseUrl, dataDir: '' }), + await openRelayDatabase({ databaseUrl: scopedDatabaseUrl, dataDir: '' }) + ) + }) + + afterAll(async () => { + for (const database of databases) await database.close() + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + } finally { + await client.end() + } + }) + + it('recovers after the cell inventory lock outlasts transaction retries', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const store = new RelayAssignmentStore(databases[0]!, () => 100) + await store.reconcileCells([cell], false) + let releaseLock: () => void = () => undefined + let reportLocked: () => void = () => undefined + const lockReleased = new Promise((resolve) => (releaseLock = resolve)) + const lockAcquired = new Promise((resolve) => (reportLocked = resolve)) + const holder = databases[1]!.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`) + reportLocked() + await lockReleased + }) + await lockAcquired + + const releaseTimer = setTimeout(releaseLock, 3_500) + try { + await reconcileCellAdmissionAtStartup({ role: 'director', cells: [cell] }, store) + } finally { + clearTimeout(releaseTimer) + releaseLock() + await holder + } + + expect(await databases[0]!.query(`SELECT cell_id FROM relay_cells`)).toEqual([ + { cell_id: cell.id } + ]) + const warnings = warn.mock.calls.flat().join('\n') + expect(warnings).toContain('orca_relay_startup_reconcile_recovered') + expect(warnings).not.toContain('orca_relay_postgres_transaction_exhausted') + }, 10_000) +}) diff --git a/cloud/apps/relay/src/cell-admission-startup.test.ts b/cloud/apps/relay/src/cell-admission-startup.test.ts new file mode 100644 index 00000000000..d08d9eefc16 --- /dev/null +++ b/cloud/apps/relay/src/cell-admission-startup.test.ts @@ -0,0 +1,119 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + reconcileCellAdmissionAtStartup, + roleOwnsAssignmentMaintenance +} from './cell-admission-startup.js' +import type { RelayCellConfig } from './config.js' + +const cells: RelayCellConfig[] = [ + { + id: 'candidate', + url: 'https://candidate.relay.example.com', + capacityRequests: 4_000, + initiallyEnabled: false + } +] + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +describe('cell admission startup authority', () => { + it('reserves global assignment maintenance for director-capable roles', () => { + expect(roleOwnsAssignmentMaintenance('cell')).toBe(false) + expect(roleOwnsAssignmentMaintenance('director')).toBe(true) + expect(roleOwnsAssignmentMaintenance('combined')).toBe(true) + }) + + it('does not let a cell process reconcile its own admission', async () => { + const reconcileCellsAtStartup = vi.fn() + await reconcileCellAdmissionAtStartup({ role: 'cell', cells }, { reconcileCellsAtStartup }) + expect(reconcileCellsAtStartup).not.toHaveBeenCalled() + }) + + it.each(['director', 'combined'] as const)( + 'lets the %s process reconcile configured admission without disabling missing cells', + async (role) => { + const reconcileCellsAtStartup = vi.fn() + await reconcileCellAdmissionAtStartup({ role, cells }, { reconcileCellsAtStartup }) + expect(reconcileCellsAtStartup).toHaveBeenCalledWith(cells) + } + ) + + it('waits out transient database pressure during director startup', async () => { + vi.useFakeTimers() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const lockUnavailable = Object.assign(new Error('lock unavailable'), { code: '55P03' }) + const reconcileCellsAtStartup = vi + .fn() + .mockRejectedValueOnce(lockUnavailable) + .mockRejectedValueOnce(lockUnavailable) + .mockResolvedValue(undefined) + + const startup = reconcileCellAdmissionAtStartup( + { role: 'director', cells }, + { reconcileCellsAtStartup } + ) + await vi.runAllTimersAsync() + await startup + + expect(reconcileCellsAtStartup).toHaveBeenCalledTimes(3) + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('orca_relay_startup_reconcile_recovered') + ) + }) + + it('fails startup immediately for a permanent reconciliation error', async () => { + const failure = new Error('invalid cell configuration') + const reconcileCellsAtStartup = vi.fn().mockRejectedValue(failure) + + await expect( + reconcileCellAdmissionAtStartup({ role: 'director', cells }, { reconcileCellsAtStartup }) + ).rejects.toBe(failure) + expect(reconcileCellsAtStartup).toHaveBeenCalledTimes(1) + }) + + it('bounds transient startup retries', async () => { + vi.useFakeTimers() + vi.spyOn(Math, 'random').mockReturnValue(0) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const lockUnavailable = Object.assign(new Error('lock unavailable'), { code: '55P03' }) + const reconcileCellsAtStartup = vi.fn().mockRejectedValue(lockUnavailable) + + const startup = reconcileCellAdmissionAtStartup( + { role: 'director', cells }, + { reconcileCellsAtStartup } + ) + const rejection = expect(startup).rejects.toBe(lockUnavailable) + await vi.runAllTimersAsync() + await rejection + + expect(reconcileCellsAtStartup).toHaveBeenCalledTimes(20) + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('orca_relay_startup_reconcile_exhausted') + ) + }) + + it('bounds retry wall time when each reconciliation is slow', async () => { + vi.useFakeTimers() + vi.setSystemTime(0) + vi.spyOn(Math, 'random').mockReturnValue(0) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const lockUnavailable = Object.assign(new Error('lock unavailable'), { code: '55P03' }) + const reconcileCellsAtStartup = vi.fn().mockImplementation(async () => { + vi.setSystemTime(Date.now() + 10_000) + throw lockUnavailable + }) + + const startup = reconcileCellAdmissionAtStartup( + { role: 'director', cells }, + { reconcileCellsAtStartup } + ) + const rejection = expect(startup).rejects.toBe(lockUnavailable) + await vi.runAllTimersAsync() + await rejection + + expect(reconcileCellsAtStartup).toHaveBeenCalledTimes(5) + }) +}) diff --git a/cloud/apps/relay/src/cell-admission-startup.ts b/cloud/apps/relay/src/cell-admission-startup.ts new file mode 100644 index 00000000000..384b6aab304 --- /dev/null +++ b/cloud/apps/relay/src/cell-admission-startup.ts @@ -0,0 +1,56 @@ +import type { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import { isRelayDatabaseTransientError } from './database.js' + +type CellAdmissionStartupConfig = Pick +type CellAdmissionStore = Pick + +const STARTUP_RECONCILE_ATTEMPTS = 20 +const STARTUP_RECONCILE_RETRY_WINDOW_MS = 45_000 +const STARTUP_RECONCILE_RETRY_BASE_MS = 250 +const STARTUP_RECONCILE_RETRY_JITTER_MS = 250 + +export function roleOwnsAssignmentMaintenance(role: RelayConfig['role']): boolean { + // Cell workers share the database but the director is the sole authority + // for global expiry, evacuation, and dead-cell maintenance. + return role !== 'cell' +} + +export async function reconcileCellAdmissionAtStartup( + config: CellAdmissionStartupConfig, + assignments: CellAdmissionStore +): Promise { + // Admission is operator/director state. A new worker must not enable itself + // before its distinct candidate has passed production preflight. + if (config.role === 'cell') return + const retryDeadline = Date.now() + STARTUP_RECONCILE_RETRY_WINDOW_MS + for (let attempt = 1; attempt <= STARTUP_RECONCILE_ATTEMPTS; attempt += 1) { + try { + await assignments.reconcileCellsAtStartup(config.cells) + if (attempt > 1) { + console.warn( + JSON.stringify({ event: 'orca_relay_startup_reconcile_recovered', attempts: attempt }) + ) + } + return + } catch (error) { + const remainingMs = retryDeadline - Date.now() + if ( + attempt === STARTUP_RECONCILE_ATTEMPTS || + remainingMs <= 0 || + !isRelayDatabaseTransientError(error) + ) { + if (isRelayDatabaseTransientError(error)) { + console.warn( + JSON.stringify({ event: 'orca_relay_startup_reconcile_exhausted', attempts: attempt }) + ) + } + throw error + } + const delayMs = + STARTUP_RECONCILE_RETRY_BASE_MS + + Math.floor(Math.random() * (STARTUP_RECONCILE_RETRY_JITTER_MS + 1)) + await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, remainingMs))) + } + } +} diff --git a/cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts b/cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts new file mode 100644 index 00000000000..56859c36132 --- /dev/null +++ b/cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts @@ -0,0 +1,75 @@ +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { + isRelayCellConnectionHardCap, + RELAY_ADMISSION_BUDGETS, + RELAY_CELL_ADMISSION_BOUNDS, + RELAY_CELL_CONNECTION_HARD_CAP, + RELAY_CELL_CONNECTION_HARD_CAPS, + relayCellAdmissionBounds +} from '@orca-cloud/relay-contract' +import { describe, expect, it } from 'vitest' + +// Why: dev scripts run standalone in CI and Terraform cannot read TypeScript, so neither can +// import the constant. Both restate it instead. This asserts every restatement still agrees, so +// raising the cap fails loudly here rather than silently leaving a surface behind. +// Repository root, and every path read below stays inside the Relay tree, so this survives the +// move under cloud/ in the public repository. +const repositoryRoot = fileURLToPath(new URL('../../../', import.meta.url)) +const read = (path: string): string => readFileSync(`${repositoryRoot}${path}`, 'utf8') + +describe('cell connection hard cap stays consistent across surfaces that cannot import it', () => { + it('derives its own bounds from the cap', () => { + expect(RELAY_CELL_ADMISSION_BOUNDS.hardCap).toBe(RELAY_CELL_CONNECTION_HARD_CAP) + expect(RELAY_CELL_ADMISSION_BOUNDS.socketAdmissionCeiling).toBe( + RELAY_CELL_CONNECTION_HARD_CAP - RELAY_ADMISSION_BUDGETS.reservedHostControls + ) + expect(RELAY_CELL_ADMISSION_BOUNDS.maxUnobservedBound).toBe( + RELAY_CELL_ADMISSION_BOUNDS.socketAdmissionCeiling - 1 + ) + for (const hardCap of RELAY_CELL_CONNECTION_HARD_CAPS) { + const bounds = relayCellAdmissionBounds(hardCap) + expect(bounds.socketAdmissionCeiling).toBe( + hardCap - RELAY_ADMISSION_BUDGETS.reservedHostControls + ) + expect(bounds.maxUnobservedBound).toBe(bounds.socketAdmissionCeiling - 1) + } + }) + + it.each([ + ['dev/scripts/relay-recovery-wave-gate.mjs', /targetConnectionCap:\s*(\d+)/], + ['dev/scripts/deploy-relay-gce-multi-target.mjs', /DEFAULT_CONNECTION_CEILING\s*=\s*(\d+)/], + ['dev/scripts/deploy-relay-gce-multi-target.mjs', /CUTOVER_CONNECTION_HARD_CAP\s*=\s*(\d+)/] + ])('%s matches the contract cap', (path, pattern) => { + const match = read(path).match(pattern) + expect(match, `${path} no longer declares ${pattern}`).not.toBeNull() + expect(Number(match![1])).toBe(RELAY_CELL_CONNECTION_HARD_CAP) + }) + + it('every production cell declaring a cap uses a supported contract cap', () => { + const declared = [ + ...read('infra/terraform/environments/production.tfvars').matchAll( + /connection_hard_cap\s*=\s*(\d+)/g + ) + ].map((match) => Number(match[1])) + expect(declared.length).toBeGreaterThan(0) + expect(new Set(declared).has(RELAY_CELL_CONNECTION_HARD_CAP)).toBe(true) + expect(declared.every((hardCap) => isRelayCellConnectionHardCap(hardCap))).toBe(true) + }) + + it('the Terraform cell validation accepts exactly the supported contract caps', () => { + const match = read('infra/terraform/variables.tf').match( + /contains\(\[([^\]]+)\], cell\.connection_hard_cap\)/ + ) + expect(match, 'variables.tf no longer validates connection_hard_cap').not.toBeNull() + expect(match![1]!.split(',').map((value) => Number(value.trim()))).toEqual([ + ...RELAY_CELL_CONNECTION_HARD_CAPS + ]) + }) + + it('the Terraform unobserved bound leaves the contract rebind reserve', () => { + expect(read('infra/terraform/variables.tf')).toContain( + 'cell.connection_unobserved_bound < cell.connection_hard_cap - 100' + ) + }) +}) diff --git a/cloud/apps/relay/src/cell-fence-legacy-adoption-postgres.test.ts b/cloud/apps/relay/src/cell-fence-legacy-adoption-postgres.test.ts new file mode 100644 index 00000000000..9fc1b915481 --- /dev/null +++ b/cloud/apps/relay/src/cell-fence-legacy-adoption-postgres.test.ts @@ -0,0 +1,122 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const cell = { + id: 'legacy-fence-adoption-postgres', + url: 'https://legacy-fence-adoption-postgres.example.com', + capacityRequests: 100 +} +const incarnation = '11111111-1111-4111-8111-111111111111' +const attemptId = '22222222-2222-4222-8222-222222222222' + +describePostgres('PostgreSQL legacy fence adoption', () => { + const databases: RelayDatabase[] = [] + + beforeAll(async () => { + databases.push( + await openRelayDatabase({ databaseUrl, dataDir: '' }), + await openRelayDatabase({ databaseUrl, dataDir: '' }) + ) + await cleanup() + }) + + afterAll(async () => { + await cleanup() + for (const database of databases) await database.close() + }) + + async function cleanup(): Promise { + const database = databases[0] + if (!database) return + await database.query( + `DELETE FROM relay_cell_fence_plan_bindings WHERE attempt_id = ?`, + [attemptId] + ) + await database.query( + `DELETE FROM relay_cell_fence_apply_invocations WHERE attempt_id = ?`, + [attemptId] + ) + await database.query(`DELETE FROM relay_cell_committed_fences WHERE cell_id = ?`, [ + cell.id + ]) + await database.query( + `DELETE FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + [cell.id] + ) + await database.query(`DELETE FROM relay_cell_fence_attempts WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_fences WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_admission WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id]) + } + + it('allows either adoption or attempt preparation, never both', async () => { + let now = 100 + const stores = databases.map( + (database) => + new RelayAssignmentStore(database, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + ) + await stores[0]!.reconcileCells([cell], false) + await stores[0]!.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: incarnation, + startedAt: 50, + ready: true, + observedRequests: 0 + }) + await stores[0]!.setCellEnabled(cell.id, false) + now += 45_001 + const evidence = { + attemptId, + environment: 'production' as const, + cellId: cell.id, + cellIncarnation: incarnation, + migName: 'orca-relay-c3', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c3', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c3-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + planObjectName: + 'terraform/state/relay-fence-plans/production/22222222-2222-4222-8222-222222222222.tfplan', + planObjectGeneration: '123456789', + varFileSha256: 'c'.repeat(64), + terraformStateLineage: '33333333-3333-4333-8333-333333333333', + terraformStateSerial: 7, + terraformStateObjectGeneration: '987654321', + terraformStateObjectSha256: 'd'.repeat(64), + requestReason: + 'orca-relay-fence/22222222-2222-4222-8222-222222222222' + } + + const results = await Promise.allSettled([ + stores[0]!.adoptLegacyCellFence(cell.id, incarnation), + stores[1]!.prepareCellFenceAttempt(evidence) + ]) + expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1) + expect(results.filter(({ status }) => status === 'rejected')).toHaveLength(1) + if (results[0]!.status === 'fulfilled') { + await stores[0]!.commitLegacyCellFenceAdoption(cell.id, incarnation) + } + const attempts = await databases[0]!.query( + `SELECT COUNT(*) AS count FROM relay_cell_fence_attempts WHERE cell_id = ?`, + [cell.id] + ) + const fences = await databases[0]!.query( + `SELECT COUNT(*) AS count FROM relay_cell_fences WHERE cell_id = ?`, + [cell.id] + ) + const adoptions = await databases[0]!.query( + `SELECT COUNT(*) AS count FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + [cell.id] + ) + expect(Number(attempts[0]!.count) + Number(fences[0]!.count)).toBe(1) + expect(Number(adoptions[0]!.count)).toBe(Number(fences[0]!.count)) + }) +}) diff --git a/cloud/apps/relay/src/cell-heartbeat-client.test.ts b/cloud/apps/relay/src/cell-heartbeat-client.test.ts new file mode 100644 index 00000000000..2aa708bed1b --- /dev/null +++ b/cloud/apps/relay/src/cell-heartbeat-client.test.ts @@ -0,0 +1,176 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RelayConfig } from './config.js' +import { startCellHeartbeat } from './cell-heartbeat-client.js' + +const CONFIG = { + role: 'cell', + cellId: 'cell-a', + cellUrl: 'https://relay-a.example.com', + directorUrl: 'https://relay.example.com', + heartbeatAudience: 'https://relay.example.com/v1/admin/cell-heartbeat' +} as RelayConfig + +describe('cell heartbeat client', () => { + afterEach(() => vi.restoreAllMocks()) + + it('sends an immediate authenticated heartbeat without putting credentials in the URL', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(url), init }) + return new Response('{}', { status: 200 }) + }) as typeof fetch + const client = startCellHeartbeat(CONFIG, { + ready: async () => true, + observedRequests: () => 7, + connectionCounts: () => ({ + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }), + fetch: fetchImpl, + identityToken: async () => 'secret-token', + now: () => 123, + incarnation: '11111111-1111-4111-8111-111111111111', + intervalMs: 60_000 + })! + await vi.waitFor(() => expect(requests).toHaveLength(1)) + client.stop() + + expect(requests[0]!.url).toBe('https://relay.example.com/v1/admin/cell-heartbeat') + expect(requests[0]!.url).not.toContain('secret-token') + expect(requests[0]!.init?.headers).toMatchObject({ authorization: 'Bearer secret-token' }) + expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({ + v: 1, + cellId: 'cell-a', + cellUrl: 'https://relay-a.example.com', + region: 'us-central1', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 123, + ready: true, + observedRequests: 7 + }) + }) + + it('advertises per-host drain only when its dedicated trust boundary is configured', async () => { + const requests: RequestInit[] = [] + const client = startCellHeartbeat( + { + ...CONFIG, + rehomeAudience: 'https://relay.example.com/v1/admin/host-drain', + rehomeDirectorServiceAccount: 'relay-director@example.com' + }, + { + ready: async () => true, + observedRequests: () => 0, + connectionCounts: () => ({ + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }), + regionalRehomeSafety: () => ({ + observedAt: 120, + sqlFailures: 0, + reconnects: 2, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }), + fetch: async (_input, init) => { + requests.push(init ?? {}) + return new Response('{}', { status: 200 }) + }, + identityToken: async () => 'secret-token', + now: () => 123, + incarnation: '11111111-1111-4111-8111-111111111111', + intervalMs: 60_000 + } + )! + await vi.waitFor(() => expect(requests).toHaveLength(2)) + client.stop() + + expect(JSON.parse(String(requests[1]!.body))).toMatchObject({ + regionalRehomeProtocol: 1, + safety: { + observedAt: 120, + sqlFailures: 0, + reconnects: 2, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + }) + }) + + it('reports the complete enforced connection ledger for a limited cell', async () => { + const requests: RequestInit[] = [] + const client = startCellHeartbeat( + { + ...CONFIG, + connectionHardCap: 600, + connectionUnobservedBound: 60 + }, + { + ready: async () => true, + observedRequests: () => 9, + connectionCounts: () => ({ + totalConnections: 10, + inFlightConnections: 2, + reservedConnectionUnits: 3, + enforcedConnectionUnits: 15, + inclusionWatermark: 42 + }), + fetch: async (_input, init) => { + requests.push(init ?? {}) + return new Response('{}', { status: 200 }) + }, + identityToken: async () => 'secret-token', + now: () => 123, + incarnation: '11111111-1111-4111-8111-111111111111', + intervalMs: 60_000 + } + )! + await vi.waitFor(() => expect(requests).toHaveLength(1)) + client.stop() + + expect(JSON.parse(String(requests[0]!.body))).toMatchObject({ + totalConnections: 10, + inFlightConnections: 2, + reservedConnectionUnits: 3, + enforcedConnectionUnits: 15, + connectionInclusionWatermark: 42, + connectionHardCap: 600, + connectionUnobservedBound: 60 + }) + }) + + it('does not start outside an explicitly configured cell role', () => { + expect( + startCellHeartbeat({ ...CONFIG, role: 'director' }, { + ready: async () => true, + observedRequests: () => 0, + connectionCounts: () => ({ + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }) + }) + ).toBeNull() + expect( + startCellHeartbeat({ ...CONFIG, directorUrl: undefined }, { + ready: async () => true, + observedRequests: () => 0, + connectionCounts: () => ({ + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }) + }) + ).toBeNull() + }) +}) diff --git a/cloud/apps/relay/src/cell-heartbeat-client.ts b/cloud/apps/relay/src/cell-heartbeat-client.ts new file mode 100644 index 00000000000..3bbcd08ecd6 --- /dev/null +++ b/cloud/apps/relay/src/cell-heartbeat-client.ts @@ -0,0 +1,123 @@ +import { randomUUID } from 'node:crypto' +import type { RelayConfig } from './config.js' +import { googleMetadataIdentityToken } from './google-metadata-identity-token.js' +import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' + +type ConnectionCounts = { + totalConnections: number + inFlightConnections: number + reservedConnectionUnits: number + enforcedConnectionUnits: number + inclusionWatermark?: number +} + +type HeartbeatClientOptions = { + ready: () => Promise + observedRequests: () => number + connectionCounts: () => ConnectionCounts + regionalRehomeSafety?: () => RegionalRehomeSafetySnapshot + fetch?: typeof fetch + identityToken?: (audience: string) => Promise + now?: () => number + incarnation?: string + intervalMs?: number +} + +export type CellHeartbeatClient = { stop: () => void; send: () => Promise } + +export function startCellHeartbeat( + config: RelayConfig, + options: HeartbeatClientOptions +): CellHeartbeatClient | null { + if (config.role !== 'cell' || !config.directorUrl || !config.heartbeatAudience) return null + const fetchImpl = options.fetch ?? fetch + const tokenProvider = + options.identityToken ?? ((audience) => googleMetadataIdentityToken(audience, fetchImpl)) + const startedAt = (options.now ?? Date.now)() + const cellIncarnation = options.incarnation ?? randomUUID() + let stopped = false + let inFlight = false + + const send = async (): Promise => { + if (stopped || inFlight) return + inFlight = true + try { + const [token, ready] = await Promise.all([ + tokenProvider(config.heartbeatAudience!), + options.ready() + ]) + const connectionCounts = + config.connectionHardCap === undefined ? null : options.connectionCounts() + const response = await fetchImpl( + new URL('/v1/admin/cell-heartbeat', config.directorUrl).toString(), + { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + cellId: config.cellId, + cellUrl: config.cellUrl, + region: config.region ?? 'us-central1', + cellIncarnation, + startedAt, + ready, + observedRequests: options.observedRequests(), + ...(config.connectionHardCap === undefined + ? {} + : { + totalConnections: connectionCounts!.totalConnections, + inFlightConnections: connectionCounts!.inFlightConnections, + reservedConnectionUnits: connectionCounts!.reservedConnectionUnits, + enforcedConnectionUnits: connectionCounts!.enforcedConnectionUnits, + connectionInclusionWatermark: + connectionCounts!.inclusionWatermark, + connectionHardCap: config.connectionHardCap, + connectionUnobservedBound: config.connectionUnobservedBound + }) + }), + signal: AbortSignal.timeout(10_000) + } + ) + if (!response.ok) throw new Error(`director_heartbeat_${response.status}`) + if (options.regionalRehomeSafety) { + const statusResponse = await fetchImpl( + new URL('/v1/admin/cell-rehome-status', config.directorUrl).toString(), + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + v: 1, + cellId: config.cellId, + cellIncarnation, + regionalRehomeProtocol: + config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0, + safety: options.regionalRehomeSafety() + }), + signal: AbortSignal.timeout(10_000) + } + ) + if (!statusResponse.ok && statusResponse.status !== 404) { + throw new Error(`director_rehome_status_${statusResponse.status}`) + } + } + } catch (error) { + // A heartbeat must fail closed without ever logging its bearer token. + console.warn('[orca-relay] cell heartbeat failed', error instanceof Error ? error.message : '') + } finally { + inFlight = false + } + } + const timer = setInterval(() => void send(), options.intervalMs ?? 15_000) + timer.unref() + void send() + return { + send, + stop: () => { + stopped = true + clearInterval(timer) + } + } +} diff --git a/cloud/apps/relay/src/config.test.ts b/cloud/apps/relay/src/config.test.ts new file mode 100644 index 00000000000..bb0522dcbd3 --- /dev/null +++ b/cloud/apps/relay/src/config.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it } from 'vitest' +import { + loadRelayConfig, + RELAY_CELL_CONNECTION_HARD_CAP, + RELAY_DATABASE_POOL_MAX, + RELAY_DIRECTOR_DATABASE_POOL_MAX, + RELAY_MAX_CELL_CAPACITY_REQUESTS, + RELAY_PUBLIC_RESOLVE_CONCURRENCY, + RELAY_PUBLIC_RESOLVE_WAIT_MS +} from './config.js' + +function cellEnvironment(capacity: number): NodeJS.ProcessEnv { + return { + ORCA_RELAY_PUBLIC_URL: 'https://c1.relay.example.com', + ORCA_RELAY_CELL_URL: 'https://c1.relay.example.com', + ORCA_RELAY_AUTH_ISSUER: 'https://auth.example.com', + ORCA_RELAY_JWKS_URL: 'https://auth.example.com/.well-known/jwks.json', + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: 'assignment-key-with-at-least-thirty-two-bytes', + ORCA_RELAY_ROLE: 'cell', + ORCA_RELAY_CELL_ID: 'gce-c1', + ORCA_RELAY_CELL_CAPACITY: String(capacity), + ORCA_RELAY_ADMIN_AUDIENCE: 'https://relay.example.com/v1/admin/drain', + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.iam.gserviceaccount.com' + } +} + +describe('GCE relay capacity configuration', () => { + it('requires distinct dedicated admin identities and accepts omitted values', () => { + const env = cellEnvironment(4_000) + expect(loadRelayConfig(env)).toMatchObject({ + capacityServiceAccount: undefined, + asiaProofServiceAccount: undefined, + monitorServiceAccount: undefined, + fenceServiceAccount: undefined + }) + env.ORCA_RELAY_MONITOR_SERVICE_ACCOUNT = '' + env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT = 'capacity@example.iam.gserviceaccount.com' + env.ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT = 'proof@example.iam.gserviceaccount.com' + env.ORCA_RELAY_FENCE_SERVICE_ACCOUNT = 'fence@example.iam.gserviceaccount.com' + expect(loadRelayConfig(env)).toMatchObject({ + capacityServiceAccount: 'capacity@example.iam.gserviceaccount.com', + asiaProofServiceAccount: 'proof@example.iam.gserviceaccount.com', + monitorServiceAccount: undefined, + fenceServiceAccount: 'fence@example.iam.gserviceaccount.com' + }) + env.ORCA_RELAY_MONITOR_SERVICE_ACCOUNT = env.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT + expect(() => loadRelayConfig(env)).toThrow('relay admin service accounts must be distinct') + env.ORCA_RELAY_MONITOR_SERVICE_ACCOUNT = undefined + env.ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT = env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT + expect(() => loadRelayConfig(env)).toThrow('relay admin service accounts must be distinct') + }) + + it('requires a distinct paired identity and exact audience for regional host drain', () => { + const env = cellEnvironment(4_000) + env.ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT = + 'relay-director@example.iam.gserviceaccount.com' + expect(() => loadRelayConfig(env)).toThrow( + 'relay rehome identity and audience must be configured together' + ) + env.ORCA_RELAY_REHOME_AUDIENCE = 'https://relay.example.com/v1/admin/host-drain' + expect(loadRelayConfig(env)).toMatchObject({ + rehomeDirectorServiceAccount: 'relay-director@example.iam.gserviceaccount.com', + rehomeAudience: 'https://relay.example.com/v1/admin/host-drain' + }) + env.ORCA_RELAY_REHOME_AUDIENCE = 'https://relay.example.com/v1/admin/drain' + expect(() => loadRelayConfig(env)).toThrow( + 'relay rehome audience must target the host drain route' + ) + env.ORCA_RELAY_REHOME_AUDIENCE = 'https://relay.example.com/v1/admin/host-drain' + env.ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT = + env.ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT + expect(() => loadRelayConfig(env)).toThrow( + 'relay rehome director identity must differ from the cell runtime identity' + ) + }) + + it('accepts a measured capacity above the Cloud Run request ceiling', () => { + expect(loadRelayConfig(cellEnvironment(4_000)).cells[0]?.capacityRequests).toBe(4_000) + }) + + it('retains a finite process-wide capacity bound', () => { + expect(() => loadRelayConfig(cellEnvironment(RELAY_MAX_CELL_CAPACITY_REQUESTS + 1))).toThrow() + }) + + it('keeps legacy cells uncapped and requires a supported hard-cap pair', () => { + const env = cellEnvironment(4_000) + expect(loadRelayConfig(env)).toMatchObject({ + connectionHardCap: undefined, + connectionUnobservedBound: undefined + }) + + env.ORCA_RELAY_CELL_CONNECTION_HARD_CAP = String(RELAY_CELL_CONNECTION_HARD_CAP) + expect(() => loadRelayConfig(env)).toThrow( + 'connection hard cap and unobserved bound must be configured together' + ) + env.ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND = '60' + expect(loadRelayConfig(env)).toMatchObject({ + connectionHardCap: 600, + connectionUnobservedBound: 60 + }) + env.ORCA_RELAY_CELL_CONNECTION_HARD_CAP = '599' + expect(() => loadRelayConfig(env)).toThrow() + env.ORCA_RELAY_CELL_CONNECTION_HARD_CAP = '600' + env.ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND = '500' + expect(() => loadRelayConfig(env)).toThrow() + env.ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND = '600' + expect(() => loadRelayConfig(env)).toThrow() + env.ORCA_RELAY_CELL_CONNECTION_HARD_CAP = '1000' + env.ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND = '60' + expect(loadRelayConfig(env)).toMatchObject({ + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + }) + env.ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND = '900' + expect(() => loadRelayConfig(env)).toThrow() + }) + + it('accepts hard-cap metadata in director cell inventory', () => { + const env = cellEnvironment(4_000) + env.ORCA_RELAY_ROLE = 'director' + env.ORCA_RELAY_PUBLIC_URL = 'https://relay.example.com' + env.ORCA_RELAY_CELL_URL = 'https://relay.example.com' + env.ORCA_RELAY_CELLS_JSON = JSON.stringify([ + { + id: 'gce-c7', + url: 'https://c7.relay.example.com', + capacityRequests: 4_000, + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + } + ]) + + expect(loadRelayConfig(env).cells[0]).toMatchObject({ + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + }) + }) + + it('accepts mixed 600- and 1000-cap director inventory', () => { + const env = cellEnvironment(4_000) + env.ORCA_RELAY_ROLE = 'director' + env.ORCA_RELAY_PUBLIC_URL = 'https://relay.example.com' + env.ORCA_RELAY_CELL_URL = 'https://relay.example.com' + env.ORCA_RELAY_CELLS_JSON = JSON.stringify([ + { + id: 'gce-c1', + url: 'https://c1.relay.example.com', + capacityRequests: 4_000, + connectionHardCap: 600, + connectionUnobservedBound: 60 + }, + { + id: 'gce-c2', + url: 'https://c2.relay.example.com', + capacityRequests: 4_000, + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + } + ]) + + expect(loadRelayConfig(env).cells.map((cell) => cell.connectionHardCap)).toEqual([ + 600, 1_000 + ]) + }) + + it('accepts only a canonical served image digest', () => { + const env = cellEnvironment(4_000) + env.ORCA_RELAY_IMAGE_DIGEST = `sha256:${'a'.repeat(64)}` + expect(loadRelayConfig(env).imageDigest).toBe(env.ORCA_RELAY_IMAGE_DIGEST) + env.ORCA_RELAY_IMAGE_DIGEST = 'relay:latest' + expect(() => loadRelayConfig(env)).toThrow() + }) + + it('accepts a statically declared candidate that starts disabled', () => { + const env = cellEnvironment(4_000) + env.ORCA_RELAY_ROLE = 'director' + env.ORCA_RELAY_PUBLIC_URL = 'https://relay.example.com' + env.ORCA_RELAY_CELL_URL = 'https://relay.example.com' + env.ORCA_RELAY_CELLS_JSON = JSON.stringify([ + { + id: 'gce-candidate', + url: 'https://candidate.relay.example.com', + capacityRequests: 4_000, + region: 'us-central1', + initiallyEnabled: false + } + ]) + expect(loadRelayConfig(env).cells).toEqual([ + { + id: 'gce-candidate', + url: 'https://candidate.relay.example.com', + capacityRequests: 4_000, + region: 'us-central1', + initiallyEnabled: false + } + ]) + }) + + it('reserves a smaller PostgreSQL connection budget for directors', () => { + const cellEnv = cellEnvironment(4_000) + expect(loadRelayConfig(cellEnv).databasePoolMax).toBe(RELAY_DATABASE_POOL_MAX) + + const directorEnv: NodeJS.ProcessEnv = { ...cellEnv, ORCA_RELAY_ROLE: 'director' } + directorEnv.ORCA_RELAY_CELLS_JSON = JSON.stringify([ + { + id: 'gce-c1', + url: 'https://c1.relay.example.com', + capacityRequests: 4_000 + } + ]) + expect(loadRelayConfig(directorEnv).databasePoolMax).toBe(RELAY_DIRECTOR_DATABASE_POOL_MAX) + + directorEnv.ORCA_RELAY_DATABASE_POOL_MAX = '7' + expect(loadRelayConfig(directorEnv).databasePoolMax).toBe(7) + }) + + it('defaults to bounded public assignment admission and supports an emergency stop', () => { + const env = cellEnvironment(4_000) + expect(loadRelayConfig(env)).toMatchObject({ + publicAssignmentsEnabled: true, + regionalPlacementEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: RELAY_PUBLIC_RESOLVE_CONCURRENCY, + publicResolveWaitMs: RELAY_PUBLIC_RESOLVE_WAIT_MS, + publicAssignmentRetryAfterSeconds: 5 + }) + + env.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED = 'false' + env.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED = 'false' + env.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY = '4' + env.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX = '256' + env.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS = '8000' + env.ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS = '30' + expect(loadRelayConfig(env)).toMatchObject({ + publicAssignmentsEnabled: false, + regionalPlacementEnabled: false, + publicAssignmentConcurrency: 4, + publicAssignmentQueueMax: 256, + publicAssignmentWaitMs: 8_000, + publicResolveConcurrency: RELAY_PUBLIC_RESOLVE_CONCURRENCY, + publicResolveWaitMs: RELAY_PUBLIC_RESOLVE_WAIT_MS, + publicAssignmentRetryAfterSeconds: 30 + }) + }) + + it('fails closed when public request lanes exceed the director database pool', () => { + const env = cellEnvironment(4_000) + env.ORCA_RELAY_ROLE = 'director' + env.ORCA_RELAY_CELLS_JSON = JSON.stringify([ + { + id: 'gce-c1', + url: 'https://c1.relay.example.com', + capacityRequests: 4_000 + } + ]) + env.ORCA_RELAY_DATABASE_POOL_MAX = '2' + + expect(() => loadRelayConfig(env)).toThrow( + 'public relay admission must leave database pool headroom' + ) + }) +}) diff --git a/cloud/apps/relay/src/config.ts b/cloud/apps/relay/src/config.ts new file mode 100644 index 00000000000..2bf23444a71 --- /dev/null +++ b/cloud/apps/relay/src/config.ts @@ -0,0 +1,348 @@ +import { z } from 'zod' +import { + isRelayCellConnectionHardCap, + RELAY_CELL_CONNECTION_HARD_CAP, + RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND, + RELAY_DEFAULT_REGION, + RelayRegionSchema, + relayCellAdmissionBounds, + type RelayCellConnectionHardCap, + type RelayRegion +} from '@orca-cloud/relay-contract' + +export const RELAY_MAX_CELL_CAPACITY_REQUESTS = 100_000 +export const RELAY_DATABASE_POOL_MAX = 10 +export const RELAY_DIRECTOR_DATABASE_POOL_MAX = 3 +export const RELAY_PUBLIC_RESOLVE_CONCURRENCY = 1 +export const RELAY_PUBLIC_RESOLVE_WAIT_MS = 5_000 +export { RELAY_CELL_CONNECTION_HARD_CAP } + +const RelayCellConnectionHardCapSchema = z.custom( + isRelayCellConnectionHardCap +) + +const EnvironmentBooleanSchema = z + .enum(['true', 'false']) + .default('true') + .transform((value) => value === 'true') + +const OptionalServiceAccountSchema = z.preprocess( + (value) => (value === '' ? undefined : value), + z.string().email().optional() +) + +const EnvSchema = z.object({ + PORT: z.coerce.number().int().positive().default(8080), + ORCA_RELAY_PUBLIC_URL: z.string().url(), + ORCA_RELAY_CELL_URL: z.string().url(), + ORCA_RELAY_AUTH_ISSUER: z.string().url(), + ORCA_RELAY_AUTH_AUDIENCE: z.literal('orca-relay').default('orca-relay'), + ORCA_RELAY_JWKS_URL: z.string().url(), + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: z.string().min(32), + ORCA_RELAY_ROLE: z.enum(['combined', 'director', 'cell']).default('combined'), + ORCA_RELAY_CELL_ID: z.string().min(1).max(128).default('combined'), + ORCA_RELAY_REGION: RelayRegionSchema.default(RELAY_DEFAULT_REGION), + ORCA_RELAY_CELL_CAPACITY: z.coerce + .number() + .int() + .positive() + .max(RELAY_MAX_CELL_CAPACITY_REQUESTS) + .default(900), + ORCA_RELAY_CELL_CONNECTION_HARD_CAP: z.coerce + .number() + .int() + .pipe(RelayCellConnectionHardCapSchema) + .optional(), + ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND: z.coerce + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND) + .optional(), + ORCA_RELAY_CELLS_JSON: z.string().default('[]'), + ORCA_RELAY_ADMIN_AUDIENCE: z.string().url(), + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: z.string().email(), + ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT: OptionalServiceAccountSchema, + ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT: OptionalServiceAccountSchema, + ORCA_RELAY_MONITOR_SERVICE_ACCOUNT: OptionalServiceAccountSchema, + ORCA_RELAY_FENCE_SERVICE_ACCOUNT: OptionalServiceAccountSchema, + ORCA_RELAY_FENCE_BROKER_SERVICE_ACCOUNT: OptionalServiceAccountSchema, + ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT: OptionalServiceAccountSchema, + ORCA_RELAY_REHOME_AUDIENCE: z.preprocess( + (value) => (value === '' ? undefined : value), + z.string().url().optional() + ), + ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: z.string().email().optional(), + ORCA_RELAY_DIRECTOR_URL: z.string().url().optional(), + ORCA_RELAY_HEARTBEAT_AUDIENCE: z.string().url().optional(), + ORCA_RELAY_IMAGE_DIGEST: z.string().regex(/^sha256:[a-f0-9]{64}$/).optional(), + ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'), + ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(), + ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema, + ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema, + ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: z.coerce.number().int().positive().max(100).default(2), + ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY: z.coerce.number().int().positive().max(100).default(1), + ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX: z.coerce.number().int().positive().max(4_096).default(64), + ORCA_RELAY_PUBLIC_STICKY_WAIT_MS: z.coerce.number().int().positive().max(30_000).default(2_000), + ORCA_RELAY_PUBLIC_STICKY_RETRY_AFTER_SECONDS: z.coerce + .number() + .int() + .positive() + .max(60) + .default(2), + ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX: z.coerce + .number() + .int() + .positive() + .max(4_096) + .default(128), + ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS: z.coerce + .number() + .int() + .positive() + .max(30_000) + .default(4_000), + ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS: z.coerce + .number() + .int() + .positive() + .max(300) + .default(5), + DATABASE_URL: z.string().optional(), + ORCA_RELAY_DATA_DIR: z.string().default('./data/relay') +}) + +const RelayCellConfigSchema = z + .object({ + id: z.string().min(1).max(128), + url: z.string().url(), + capacityRequests: z.number().int().positive().max(RELAY_MAX_CELL_CAPACITY_REQUESTS), + region: RelayRegionSchema.default(RELAY_DEFAULT_REGION), + initiallyEnabled: z.boolean().optional(), + connectionHardCap: RelayCellConnectionHardCapSchema.optional(), + connectionUnobservedBound: z + .number() + .int() + .nonnegative() + .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND) + .optional() + }) + .strict() + .superRefine((value, context) => { + if ( + (value.connectionHardCap === undefined) !== + (value.connectionUnobservedBound === undefined) + ) { + context.addIssue({ + code: 'custom', + message: 'connection hard cap and unobserved bound must be configured together' + }) + } else if ( + value.connectionHardCap !== undefined && + value.connectionUnobservedBound! > + relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound + ) { + context.addIssue({ + code: 'custom', + path: ['connectionUnobservedBound'], + message: 'connection unobserved bound must leave ordinary admission capacity' + }) + } + }) + +export type RelayCellConfig = Omit, 'region'> & { + region?: RelayRegion +} + +export type RelayConfig = { + port: number + publicUrl: string + cellUrl: string + authIssuer: string + authAudience: 'orca-relay' + jwksUrl: string + assignmentSigningKey: Uint8Array + role: 'combined' | 'director' | 'cell' + cellId: string + region?: RelayRegion + cells: RelayCellConfig[] + adminAudience: string + deployServiceAccount: string + capacityServiceAccount?: string + asiaProofServiceAccount?: string + monitorServiceAccount?: string + fenceServiceAccount?: string + fenceBrokerServiceAccount?: string + rehomeDirectorServiceAccount?: string + rehomeAudience?: string + runtimeServiceAccount: string + directorUrl?: string + heartbeatAudience?: string + imageDigest?: string + connectionHardCap?: RelayCellConnectionHardCap + connectionUnobservedBound?: number + adminJwksUrl: string + databasePoolMax: number + publicAssignmentsEnabled: boolean + regionalPlacementEnabled?: boolean + publicAssignmentConcurrency: number + publicAssignmentQueueMax: number + publicAssignmentWaitMs: number + publicResolveConcurrency: number + publicResolveWaitMs: number + publicAssignmentRetryAfterSeconds: number + publicStickyConcurrency?: number + publicStickyQueueMax?: number + publicStickyWaitMs?: number + publicStickyRetryAfterSeconds?: number + databaseUrl?: string + dataDir: string +} + +function canonicalOrigin(value: string, name: string): string { + const url = new URL(value) + if (url.origin !== value || url.pathname !== '/') throw new Error(`${name} must be an origin`) + const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname) + if (url.protocol !== 'https:' && !(loopback && url.protocol === 'http:')) { + throw new Error(`${name} must use HTTPS outside loopback development`) + } + return value +} + +export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConfig { + const parsed = EnvSchema.parse(env) + const adminServiceAccounts = [ + parsed.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT, + parsed.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT, + parsed.ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT, + parsed.ORCA_RELAY_MONITOR_SERVICE_ACCOUNT, + parsed.ORCA_RELAY_FENCE_SERVICE_ACCOUNT, + parsed.ORCA_RELAY_FENCE_BROKER_SERVICE_ACCOUNT, + parsed.ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT + ].filter((value): value is string => value !== undefined) + if (new Set(adminServiceAccounts).size !== adminServiceAccounts.length) { + throw new Error('relay admin service accounts must be distinct') + } + if ( + (parsed.ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT === undefined) !== + (parsed.ORCA_RELAY_REHOME_AUDIENCE === undefined) + ) { + throw new Error('relay rehome identity and audience must be configured together') + } + if ( + parsed.ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT && + parsed.ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT === + (parsed.ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT ?? parsed.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT) + ) { + throw new Error('relay rehome director identity must differ from the cell runtime identity') + } + if ( + parsed.ORCA_RELAY_REHOME_AUDIENCE && + new URL(parsed.ORCA_RELAY_REHOME_AUDIENCE).pathname !== '/v1/admin/host-drain' + ) { + throw new Error('relay rehome audience must target the host drain route') + } + const directorUrl = parsed.ORCA_RELAY_DIRECTOR_URL + ? canonicalOrigin(parsed.ORCA_RELAY_DIRECTOR_URL, 'ORCA_RELAY_DIRECTOR_URL') + : undefined + const publicUrl = canonicalOrigin(parsed.ORCA_RELAY_PUBLIC_URL, 'ORCA_RELAY_PUBLIC_URL') + const configuredCells = z + .array(RelayCellConfigSchema) + .max(128) + .parse(JSON.parse(parsed.ORCA_RELAY_CELLS_JSON) as unknown) + .map((cell) => ({ ...cell, url: canonicalOrigin(cell.url, `cell ${cell.id}`) })) + const ownCell = { + id: parsed.ORCA_RELAY_CELL_ID, + region: parsed.ORCA_RELAY_REGION, + url: canonicalOrigin(parsed.ORCA_RELAY_CELL_URL, 'ORCA_RELAY_CELL_URL'), + capacityRequests: parsed.ORCA_RELAY_CELL_CAPACITY, + connectionHardCap: parsed.ORCA_RELAY_CELL_CONNECTION_HARD_CAP, + connectionUnobservedBound: parsed.ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND, + initiallyEnabled: true + } + if ( + (ownCell.connectionHardCap === undefined) !== + (ownCell.connectionUnobservedBound === undefined) + ) { + throw new Error('connection hard cap and unobserved bound must be configured together') + } + if ( + ownCell.connectionHardCap !== undefined && + ownCell.connectionUnobservedBound! > + relayCellAdmissionBounds(ownCell.connectionHardCap).maxUnobservedBound + ) { + throw new Error('connection unobserved bound must leave ordinary admission capacity') + } + const cells = parsed.ORCA_RELAY_ROLE === 'director' ? configuredCells : [ownCell] + if (cells.length === 0) throw new Error('director requires at least one configured cell') + if (new Set(cells.map(({ id }) => id)).size !== cells.length) { + throw new Error('relay cell ids must be unique') + } + const databasePoolMax = + parsed.ORCA_RELAY_DATABASE_POOL_MAX ?? + (parsed.ORCA_RELAY_ROLE === 'director' + ? RELAY_DIRECTOR_DATABASE_POOL_MAX + : RELAY_DATABASE_POOL_MAX) + if ( + parsed.ORCA_RELAY_ROLE !== 'cell' && + parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY >= databasePoolMax + ) { + throw new Error('public relay admission must leave database pool headroom') + } + if ( + parsed.ORCA_RELAY_ROLE !== 'cell' && + parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY + parsed.ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY > + databasePoolMax + ) { + throw new Error('sticky and placement admission together must fit the database pool') + } + return { + port: parsed.PORT, + publicUrl, + cellUrl: ownCell.url, + authIssuer: canonicalOrigin(parsed.ORCA_RELAY_AUTH_ISSUER, 'ORCA_RELAY_AUTH_ISSUER'), + authAudience: parsed.ORCA_RELAY_AUTH_AUDIENCE, + jwksUrl: parsed.ORCA_RELAY_JWKS_URL, + assignmentSigningKey: new TextEncoder().encode(parsed.ORCA_RELAY_ASSIGNMENT_SIGNING_KEY), + role: parsed.ORCA_RELAY_ROLE, + cellId: ownCell.id, + region: ownCell.region, + cells, + adminAudience: parsed.ORCA_RELAY_ADMIN_AUDIENCE, + deployServiceAccount: parsed.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT, + capacityServiceAccount: parsed.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT, + asiaProofServiceAccount: parsed.ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT, + monitorServiceAccount: parsed.ORCA_RELAY_MONITOR_SERVICE_ACCOUNT, + fenceServiceAccount: parsed.ORCA_RELAY_FENCE_SERVICE_ACCOUNT, + fenceBrokerServiceAccount: parsed.ORCA_RELAY_FENCE_BROKER_SERVICE_ACCOUNT, + rehomeDirectorServiceAccount: parsed.ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT, + rehomeAudience: parsed.ORCA_RELAY_REHOME_AUDIENCE, + runtimeServiceAccount: + parsed.ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT ?? parsed.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT, + directorUrl, + heartbeatAudience: + parsed.ORCA_RELAY_HEARTBEAT_AUDIENCE ?? + (directorUrl || parsed.ORCA_RELAY_ROLE === 'director' + ? new URL('/v1/admin/cell-heartbeat', directorUrl ?? publicUrl).toString() + : undefined), + imageDigest: parsed.ORCA_RELAY_IMAGE_DIGEST, + connectionHardCap: ownCell.connectionHardCap, + connectionUnobservedBound: ownCell.connectionUnobservedBound, + adminJwksUrl: parsed.ORCA_RELAY_ADMIN_JWKS_URL, + databasePoolMax, + publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED, + regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED, + publicAssignmentConcurrency: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY, + publicAssignmentQueueMax: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX, + publicAssignmentWaitMs: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS, + publicResolveConcurrency: RELAY_PUBLIC_RESOLVE_CONCURRENCY, + publicResolveWaitMs: RELAY_PUBLIC_RESOLVE_WAIT_MS, + publicAssignmentRetryAfterSeconds: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS, + publicStickyConcurrency: parsed.ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY, + publicStickyQueueMax: parsed.ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX, + publicStickyWaitMs: parsed.ORCA_RELAY_PUBLIC_STICKY_WAIT_MS, + publicStickyRetryAfterSeconds: parsed.ORCA_RELAY_PUBLIC_STICKY_RETRY_AFTER_SECONDS, + databaseUrl: parsed.DATABASE_URL, + dataDir: parsed.ORCA_RELAY_DATA_DIR + } +} diff --git a/cloud/apps/relay/src/connection-reservation-debt-retention.test.ts b/cloud/apps/relay/src/connection-reservation-debt-retention.test.ts new file mode 100644 index 00000000000..fd50f9d6747 --- /dev/null +++ b/cloud/apps/relay/src/connection-reservation-debt-retention.test.ts @@ -0,0 +1,130 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayCellConfig } from './config.js' +import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js' + +const DEBT_RETENTION_MS = 10 * 60 * 1_000 + +const LIMITED_CELL: RelayCellConfig = { + id: 'limited', + url: 'https://limited.example.com', + capacityRequests: 1_000, + connectionHardCap: 600, + connectionUnobservedBound: 50 +} + +const databases: RelayDatabase[] = [] + +afterEach(async () => { + for (const database of databases.splice(0)) await database.close() +}) + +async function setup(now: () => number): Promise<{ + database: RelayDatabase + store: RelayAssignmentStore +}> { + const database = await openInMemoryRelayDatabase() + databases.push(database) + const store = new RelayAssignmentStore(database, now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells([LIMITED_CELL], true) + await store.recordCellHeartbeat({ + cellId: LIMITED_CELL.id, + cellUrl: LIMITED_CELL.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionHardCap: 600, + connectionUnobservedBound: 50 + }) + return { database, store } +} + +async function insertDebt( + database: RelayDatabase, + reservationId: string, + relayHostId: string, + timeoutAt: number, + claimActivityId: string | null = null +): Promise { + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch, + cell_id, state, claim_activity_id, created_at, timeout_at, updated_at) + VALUES (?, ?, 'user-1', ?, 1, ?, 'late-arrival-debt', ?, ?, ?, ?)`, + [ + reservationId, + reservationId, + relayHostId, + LIMITED_CELL.id, + claimActivityId, + timeoutAt - 10_000, + timeoutAt, + timeoutAt + ] + ) +} + +async function reservationStates(database: RelayDatabase): Promise> { + const rows = await database.query( + `SELECT reservation_id, state FROM relay_control_connection_reservations` + ) + return new Map(rows.map((row) => [String(row['reservation_id']), String(row['state'])])) +} + +describe('late-arrival debt retention', () => { + it('releases unclaimed debt past retention and keeps fresh or claimed debt', async () => { + const now = 100_000_000 + const { database, store } = await setup(() => now) + await insertDebt(database, 'stale-debt', 'host000000000001', now - DEBT_RETENTION_MS - 1) + await insertDebt(database, 'fresh-debt', 'host000000000002', now - 30_000) + await insertDebt( + database, + 'claimed-debt', + 'host000000000003', + now - DEBT_RETENTION_MS - 1, + 'control:1' + ) + + await store.releaseExpiredActivityLeases() + + expect(await reservationStates(database)).toEqual( + new Map([ + ['stale-debt', 'released'], + ['fresh-debt', 'late-arrival-debt'], + ['claimed-debt', 'late-arrival-debt'] + ]) + ) + }) + + it('restores placement once stale debt no longer consumes connection headroom', async () => { + const now = 100_000_000 + const { database, store } = await setup(() => now) + // Headroom needs enforced + outstanding + unobserved(50) < hardCap(600) - + // rebindReserve(100); 450 stale debt rows are exactly enough to block. + for (let index = 0; index < 450; index++) { + await insertDebt( + database, + `stale-${index}`, + `host${String(index).padStart(12, '0')}`, + now - DEBT_RETENTION_MS - 1 + ) + } + await expect( + store.assign({ userId: 'user-9', relayHostId: 'hostfffffffffff9' }) + ).rejects.toThrow('relay_capacity_exhausted') + + await store.releaseExpiredActivityLeases() + + await expect( + store.assign({ userId: 'user-9', relayHostId: 'hostfffffffffff9' }) + ).resolves.toMatchObject({ cellId: LIMITED_CELL.id }) + }) +}) diff --git a/cloud/apps/relay/src/control-lease-recovery-postgres.test.ts b/cloud/apps/relay/src/control-lease-recovery-postgres.test.ts new file mode 100644 index 00000000000..2fa01c27df8 --- /dev/null +++ b/cloud/apps/relay/src/control-lease-recovery-postgres.test.ts @@ -0,0 +1,223 @@ +import { EventEmitter } from 'node:events' +import { ASSIGNMENT_LIMITS, RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import type WebSocket from 'ws' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import type { RelayCredentialStore } from './credential-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' +import { HostSessionRegistry, type HostSession } from './host-session-registry.js' +import type { RelayRuntimeObserver } from './relay-observability.js' +import type { RelayTokenClaims } from './relay-token-verifier.js' +import { ProcessQueuedByteBudget } from './splice-forwarder.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip + +const sourceCell = { + id: 'control-recovery-source', + url: 'https://control-recovery-source.example.com', + capacityRequests: 100 +} +const targetCell = { + id: 'control-recovery-target', + url: 'https://control-recovery-target.example.com', + capacityRequests: 100 +} +const userId = 'control-recovery-user' +const identities = ['controlrecovery1', 'controlrecovery2'].map((relayHostId) => ({ + userId, + relayHostId +})) + +class FakeSocket extends EventEmitter { + readonly OPEN = 1 + readonly CLOSED = 3 + readyState = this.OPEN + readonly send = vi.fn() + readonly close = vi.fn((code?: number, reason?: string) => { + this.readyState = this.CLOSED + this.emit('close', code, Buffer.from(reason ?? '')) + }) +} + +const observer = { + recordAuth: vi.fn(), + recordForwardedBytes: vi.fn(), + recordHttp: vi.fn(), + recordReconnect: vi.fn(), + recordSql: vi.fn() +} satisfies RelayRuntimeObserver + +type RegistryInternals = { + activate( + socket: WebSocket, + identity: RelayTokenClaims, + existing: HostSession | null, + generation: number, + rebind: boolean, + assignmentEpoch: number, + appVersion: string + ): Promise + heartbeat(session: HostSession): void +} + +describePostgres('expired control lease after a database outage', () => { + let database: RelayDatabase + let now = 1_900_000_000_000 + + const removeFixtureRows = async (): Promise => { + await database.query(`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`, [ + userId + ]) + await database.query(`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, [userId]) + await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [userId]) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [userId]) + for (const cell of [sourceCell, targetCell]) { + await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id]) + } + } + + const createRegistry = (store: RelayAssignmentStore): HostSessionRegistry => + new HostSessionRegistry( + { + role: 'cell', + cellId: sourceCell.id + } as RelayConfig, + vi.fn(), + {} as RelayCredentialStore, + store, + new ProcessQueuedByteBudget(), + observer, + () => now + ) + + const activate = async ( + registry: HostSessionRegistry, + store: RelayAssignmentStore, + index: number + ): Promise<{ activityId: string; session: HostSession; socket: FakeSocket }> => { + const identity = identities[index]! + const assignment = await store.assign(identity) + const socket = new FakeSocket() + const token = { + sub: identity.userId, + prof: 'control-recovery-profile', + relayHostId: identity.relayHostId, + purpose: 'host-control', + exp: 4_102_444_800 + } satisfies RelayTokenClaims + await (registry as unknown as RegistryInternals).activate( + socket as unknown as WebSocket, + token, + null, + 1, + false, + assignment.assignmentEpoch, + 'control-recovery-test' + ) + const session = registry.get(identity)! + clearInterval(session.heartbeatTimer!) + session.heartbeatTimer = null + return { activityId: session.controlActivityId!, session, socket } + } + + const heartbeat = (registry: HostSessionRegistry, session: HostSession): void => { + session.activityRenewalDueAt = now + session.lastPongAt = now + const internals = registry as unknown as RegistryInternals + internals.heartbeat(session) + } + + const leaseRows = async (relayHostId: string) => + await database.query( + `SELECT activity_id, cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [userId, relayHostId] + ) + + const waitForRenewal = async ( + socket: FakeSocket, + relayHostId: string, + expectedRows: number + ): Promise => { + await expect + .poll( + async () => + socket.close.mock.calls.length > 0 || + (await leaseRows(relayHostId)).length === expectedRows + ) + .toBe(true) + } + + beforeAll(async () => { + database = await openRelayDatabase({ databaseUrl, dataDir: '' }) + }) + + beforeEach(async () => { + now = 1_900_000_000_000 + await removeFixtureRows() + }) + + afterEach(async () => { + await removeFixtureRows() + }) + + afterAll(async () => { + if (database) await database.close() + }) + + it('re-acquires a reaped lease while the assignment remains on this cell', async () => { + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells([sourceCell]) + const registry = createRegistry(store) + const { activityId, session, socket } = await activate(registry, store, 0) + + now += ASSIGNMENT_LIMITS.activityLeaseMs + 1 + await store.releaseExpiredActivityLeases() + expect(await leaseRows(identities[0]!.relayHostId)).toHaveLength(0) + + heartbeat(registry, session) + await waitForRenewal(socket, identities[0]!.relayHostId, 1) + + expect(socket.close).not.toHaveBeenCalled() + expect(await leaseRows(identities[0]!.relayHostId)).toEqual([ + expect.objectContaining({ activity_id: activityId, cell_id: sourceCell.id }) + ]) + registry.drain(0) + }) + + it('closes without stealing back a lease that moved to another cell', async () => { + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells([sourceCell, targetCell]) + const registry = createRegistry(store) + const { activityId, session, socket } = await activate(registry, store, 1) + const identity = identities[1]! + await database.query( + `UPDATE relay_assignment_activity_leases SET cell_id = ? + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [targetCell.id, identity.userId, identity.relayHostId, activityId] + ) + + await expect( + store.renewControlActivity(identity, { + activityId, + cellId: sourceCell.id, + expiresAt: now + ASSIGNMENT_LIMITS.activityLeaseMs + }) + ).rejects.toThrow('control_activity_moved') + + heartbeat(registry, session) + await expect.poll(() => socket.close.mock.calls.length).toBe(1) + + expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved') + expect(await leaseRows(identity.relayHostId)).toEqual([ + expect.objectContaining({ activity_id: activityId, cell_id: targetCell.id }) + ]) + registry.drain(0) + }) +}) diff --git a/cloud/apps/relay/src/control-renewal-postgres.test.ts b/cloud/apps/relay/src/control-renewal-postgres.test.ts new file mode 100644 index 00000000000..fb49e3af3a2 --- /dev/null +++ b/cloud/apps/relay/src/control-renewal-postgres.test.ts @@ -0,0 +1,349 @@ +import { ASSIGNMENT_LIMITS, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { + openRelayDatabase, + type RelayDatabase, + type RelayLockOptions, + type SqlRow +} from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip + +const sourceCell = { + id: 'control-renewal-source', + url: 'https://control-renewal-source.example.com', + capacityRequests: 100 +} +const targetCell = { + id: 'control-renewal-target', + url: 'https://control-renewal-target.example.com', + capacityRequests: 100 +} +const userId = 'control-renewal-postgres-user' +const identities = Array.from({ length: 6 }, (_, index) => ({ + userId, + relayHostId: `controlrenewal${index + 1}` +})) + +function signal(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + return { promise: new Promise((done) => (resolve = done)), resolve } +} + +class StallFirstTransactionDatabase implements RelayDatabase { + readonly stalled = signal() + readonly continue = signal() + private stallNext = true + + constructor(private readonly database: RelayDatabase) {} + + async query(sql: string, params?: unknown[]): Promise { + return await this.database.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + return await this.database.queryLocked(sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + if (this.stallNext) { + this.stallNext = false + this.stalled.resolve() + await this.continue.promise + } + return await this.database.transaction(operation) + } + + async close(): Promise {} +} + +class StallFirstRenewalQueryDatabase implements RelayDatabase { + readonly dialect = 'postgres' as const + readonly stalled = signal() + readonly continue = signal() + private stallNext = true + + constructor(private readonly database: RelayDatabase) {} + + async query(sql: string, params?: unknown[]): Promise { + if (this.stallNext && sql.includes('WITH assignment_state AS MATERIALIZED')) { + this.stallNext = false + this.stalled.resolve() + await this.continue.promise + } + return await this.database.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + return await this.database.queryLocked(sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await this.database.transaction(operation) + } + + async close(): Promise {} +} + +class RenewalQueryProbeDatabase implements RelayDatabase { + readonly dialect = 'postgres' as const + renewalQueries = 0 + transactions = 0 + + constructor(private readonly database: RelayDatabase) {} + + async query(sql: string, params?: unknown[]): Promise { + if (sql.includes('WITH assignment_state AS MATERIALIZED')) this.renewalQueries++ + return await this.database.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + return await this.database.queryLocked(sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + this.transactions++ + return await this.database.transaction(operation) + } + + async close(): Promise {} +} + +describePostgres('PostgreSQL control renewal', () => { + let database: RelayDatabase + let now = 1_900_000_000_000 + const controlId = (cellId: string): string => `control:${cellId}:1` + + const removeFixtureRows = async (): Promise => { + await database.query(`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`, [ + userId + ]) + await database.query(`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, [userId]) + await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [userId]) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [userId]) + for (const cell of [sourceCell, targetCell]) { + await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id]) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id]) + } + } + + beforeAll(async () => { + database = await openRelayDatabase({ databaseUrl, dataDir: '' }) + await removeFixtureRows() + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells([sourceCell]) + for (const identity of identities) { + const assignment = await store.assign(identity) + expect(assignment.cellId).toBe(sourceCell.id) + await store.activateControl(identity, { + cellId: sourceCell.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + } + await store.reconcileCells([sourceCell, targetCell]) + }) + + afterAll(async () => { + if (!database) return + await removeFixtureRows() + await database.close() + }) + + it('reaches PostgreSQL past an acquireActivity stalled in the identity queue', async () => { + const probe = new StallFirstTransactionDatabase(database) + const store = new RelayAssignmentStore(probe, () => now) + const identity = identities[0]! + const queued = store.acquireActivity(identity, { + activityId: 'splice:queue-blocker', + kind: 'splice', + cellId: sourceCell.id + }) + await probe.stalled.promise + const expiresAt = now + 105_000 + + try { + await Promise.race([ + store.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt + }), + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error('renewal_waited_for_identity_queue')), 2_000) + ) + ]) + const row = ( + await database.query( + `SELECT expires_at FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, controlId(sourceCell.id)] + ) + )[0] + expect(Number(row!.expires_at)).toBe(expiresAt) + } finally { + probe.continue.resolve() + await queued + } + }) + + it('does not let a late older renewal rewind lease or activity timestamps', async () => { + const probe = new StallFirstRenewalQueryDatabase(database) + const store = new RelayAssignmentStore(probe, () => now) + const identity = identities[1]! + const earlierNow = now + const earlierExpiry = earlierNow + 105_000 + const earlier = store.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt: earlierExpiry + }) + await probe.stalled.promise + + now += 30_000 + const laterNow = now + const laterExpiry = laterNow + 105_000 + await store.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt: laterExpiry + }) + probe.continue.resolve() + await earlier + + const lease = ( + await database.query( + `SELECT expires_at, updated_at FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, controlId(sourceCell.id)] + ) + )[0] + const assignment = ( + await database.query( + `SELECT lease_expires_at, last_activity_at FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0] + expect(lease).toMatchObject({ + expires_at: String(laterExpiry), + updated_at: String(laterNow) + }) + expect(assignment).toMatchObject({ + lease_expires_at: String(laterExpiry), + last_activity_at: String(laterNow) + }) + }) + + it('allows the source only while its exact forward migration remains active', async () => { + const identity = identities[2]! + const store = new RelayAssignmentStore(database, () => now) + const migration = await store.startEvacuation(identity, targetCell.id) + const activeExpiry = now + 105_000 + + await expect( + store.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt: activeExpiry + }) + ).resolves.toBeUndefined() + + await database.query( + `UPDATE relay_assignment_migrations SET completed_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, identity.userId, identity.relayHostId, migration.assignmentEpoch] + ) + now += 30_000 + await expect( + store.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt: now + 105_000 + }) + ).rejects.toThrow('activity_cell_not_authoritative') + + const lease = ( + await database.query( + `SELECT expires_at FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, controlId(sourceCell.id)] + ) + )[0] + expect(Number(lease!.expires_at)).toBe(activeExpiry) + }) + + it('does not resurrect a control released while its renewal was stalled', async () => { + const probe = new StallFirstRenewalQueryDatabase(database) + const renewalStore = new RelayAssignmentStore(probe, () => now) + const releaseStore = new RelayAssignmentStore(database, () => now) + const identity = identities[3]! + const renewal = renewalStore.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt: now + 105_000 + }) + await probe.stalled.promise + + await expect(releaseStore.releaseActivity(identity, controlId(sourceCell.id))).resolves.toBe( + true + ) + probe.continue.resolve() + await expect(renewal).rejects.toThrow('control_activity_not_found') + const rows = await database.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, controlId(sourceCell.id)] + ) + expect(rows).toHaveLength(0) + }) + + it('rejects a renewal expiry beyond the maximum control lease horizon', async () => { + const identity = identities[4]! + const store = new RelayAssignmentStore(database, () => now) + const maximumExpiry = + now + + ASSIGNMENT_LIMITS.activityLeaseMs + + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2 + + await expect( + store.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt: maximumExpiry + 1 + }) + ).rejects.toThrow('invalid_activity_expiry') + }) + + it('uses one autocommitted PostgreSQL statement for a steady renewal', async () => { + const probe = new RenewalQueryProbeDatabase(database) + const store = new RelayAssignmentStore(probe, () => now) + const identity = identities[5]! + + await store.renewControlActivity(identity, { + activityId: controlId(sourceCell.id), + cellId: sourceCell.id, + expiresAt: now + 105_000 + }) + + expect(probe.renewalQueries).toBe(1) + expect(probe.transactions).toBe(0) + }) +}) diff --git a/cloud/apps/relay/src/control-reservation-reconnect-claim.test.ts b/cloud/apps/relay/src/control-reservation-reconnect-claim.test.ts new file mode 100644 index 00000000000..e6fb44e336c --- /dev/null +++ b/cloud/apps/relay/src/control-reservation-reconnect-claim.test.ts @@ -0,0 +1,201 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayCellConfig } from './config.js' +import { + openInMemoryRelayDatabase, + openRelayDatabase, + type RelayDatabase +} from './database.js' + +const KEY_PREFIX = 'reconnect-claim' +const USER_ID = `${KEY_PREFIX}-user-1` +const RELAY_HOST_ID = 'reconnectclaim01' +const CELL: RelayCellConfig = { + id: `${KEY_PREFIX}-cell`, + url: `https://${KEY_PREFIX}-cell.example.com`, + capacityRequests: 1_000, + connectionHardCap: 600, + connectionUnobservedBound: 50 +} +const CELL_INCARNATION = '11111111-1111-4111-8111-111111111111' +const IDENTITY = { userId: USER_ID, relayHostId: RELAY_HOST_ID } +const CONTROL_ACTIVITY_ID = `control:${CELL.id}:1` + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const backends: { name: string; open: () => Promise }[] = [ + { name: 'sqlite', open: openInMemoryRelayDatabase }, + ...(databaseUrl + ? [ + { + name: 'postgres', + open: () => openRelayDatabase({ databaseUrl, dataDir: '' }) + } + ] + : []) +] + +const databases: RelayDatabase[] = [] + +async function removeScopedRows(database: RelayDatabase): Promise { + await database.query( + `DELETE FROM relay_control_connection_reservations WHERE user_id LIKE '${KEY_PREFIX}-%'` + ) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE '${KEY_PREFIX}-%'` + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id LIKE '${KEY_PREFIX}-%'`) + for (const table of [ + 'relay_cell_connection_snapshots', + 'relay_cell_connection_runtime', + 'relay_cell_connection_limits', + 'relay_cell_runtime', + 'relay_cells' + ]) { + await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [CELL.id]) + } +} + +afterEach(async () => { + for (const database of databases.splice(0)) { + await removeScopedRows(database) + await database.close() + } +}) + +async function setup( + open: () => Promise, + now: () => number +): Promise<{ database: RelayDatabase; store: RelayAssignmentStore }> { + const database = await open() + databases.push(database) + await removeScopedRows(database) + const store = new RelayAssignmentStore(database, now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells([CELL], false) + await heartbeat(store, 0) + return { database, store } +} + +async function heartbeat(store: RelayAssignmentStore, watermark: number): Promise { + await store.recordCellHeartbeat({ + cellId: CELL.id, + cellUrl: CELL.url, + cellIncarnation: CELL_INCARNATION, + startedAt: 50, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionHardCap: CELL.connectionHardCap, + connectionUnobservedBound: CELL.connectionUnobservedBound, + connectionInclusionWatermark: watermark + }) +} + +async function reservations( + database: RelayDatabase +): Promise<{ state: string; claimActivityId: string | null }[]> { + const rows = await database.query( + `SELECT state, claim_activity_id FROM relay_control_connection_reservations + WHERE user_id = ? ORDER BY created_at ASC, reservation_id ASC`, + [USER_ID] + ) + return rows.map((row) => ({ + state: String(row['state']), + claimActivityId: + row['claim_activity_id'] === null || row['claim_activity_id'] === undefined + ? null + : String(row['claim_activity_id']) + })) +} + +describe.each(backends)('control reservation claim on reconnect ($name)', ({ open }) => { + it('claims the fresh reservation when the same generation reconnects', async () => { + let now = 100_000_000 + const { database, store } = await setup(open, () => now) + + const assignment = await store.assign(IDENTITY) + await store.activateControl(IDENTITY, { + cellId: CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1, + connectionInclusionWatermark: 1 + }) + // Cell telemetry now includes the control, so its reservation is released. + now += 1_000 + await heartbeat(store, 2) + expect(await reservations(database)).toEqual([ + { state: 'released', claimActivityId: CONTROL_ACTIVITY_ID } + ]) + + // Control drops and the host is re-granted the same cell and epoch. + now += 1_000 + await store.releaseActivity(IDENTITY, CONTROL_ACTIVITY_ID) + now += 1_000 + await store.assign(IDENTITY) + expect(await reservations(database)).toEqual([ + { state: 'released', claimActivityId: CONTROL_ACTIVITY_ID }, + { state: 'reserved', claimActivityId: null } + ]) + + now += 1_000 + await store.activateControl(IDENTITY, { + cellId: CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1, + connectionInclusionWatermark: 3 + }) + + expect(await reservations(database)).toEqual([ + { state: 'released', claimActivityId: CONTROL_ACTIVITY_ID }, + { state: 'claimed', claimActivityId: CONTROL_ACTIVITY_ID } + ]) + }) + + it('still refuses to claim a second reservation while the first is outstanding', async () => { + let now = 100_000_000 + const { database, store } = await setup(open, () => now) + + const assignment = await store.assign(IDENTITY) + await store.activateControl(IDENTITY, { + cellId: CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1, + connectionInclusionWatermark: 1 + }) + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch, + cell_id, state, claim_activity_id, created_at, timeout_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'reserved', NULL, ?, ?, ?)`, + [ + `${KEY_PREFIX}-extra`, + `${KEY_PREFIX}-extra`, + USER_ID, + RELAY_HOST_ID, + assignment.assignmentEpoch, + CELL.id, + now + 1, + now + 90_000, + now + 1 + ] + ) + + now += 1_000 + await store.activateControl(IDENTITY, { + cellId: CELL.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1, + connectionInclusionWatermark: 2 + }) + + expect(await reservations(database)).toEqual([ + { state: 'claimed', claimActivityId: CONTROL_ACTIVITY_ID }, + { state: 'reserved', claimActivityId: null } + ]) + }) +}) diff --git a/cloud/apps/relay/src/credential-store.test.ts b/cloud/apps/relay/src/credential-store.test.ts new file mode 100644 index 00000000000..9f279b93791 --- /dev/null +++ b/cloud/apps/relay/src/credential-store.test.ts @@ -0,0 +1,301 @@ +import { describe, expect, it } from 'vitest' +import { + hashCredential, + RelayCredentialStore, + type CredentialReservation, + type RelayIdentity +} from './credential-store.js' +import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js' + +const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } +const relayDeviceId = 'device-1' + +async function inviteBasis( + store: RelayCredentialStore, + generation = 1 +): Promise<{ reservation: CredentialReservation; basisConnId: string }> { + const invite = await store.createInvite(identity, relayDeviceId) + const reservation = await store.reserveCredential(identity.relayHostId, invite.inviteToken) + if (!reservation) throw new Error('invite did not reserve') + const basisConnId = `basis-${generation}` + await store.recordConnectionBasis({ + ...reservation, + basisConnId, + owningControlGeneration: generation, + deadline: reservation.leaseExpiresAt + }) + return { reservation, basisConnId } +} + +describe('relay credential store', () => { + it('issues invites with a skew margin under the client-side TTL ceiling', async () => { + const database = await openInMemoryRelayDatabase() + const store = new RelayCredentialStore(database, () => 1_000_000) + const invite = await store.createInvite(identity, relayDeviceId) + // Zero-tolerance released desktops reject expiry past local now + 10min; + // issuing 30s under the ceiling absorbs that much desktop clock lag. + expect(invite.expiresAt).toBe(1_000_000 + 10 * 60 * 1_000 - 30 * 1_000) + }) + + it('persists one invite reservation with bounded attempts and cooldown', async () => { + let now = 100 + const database = await openInMemoryRelayDatabase() + const store = new RelayCredentialStore(database, () => now) + const invite = await store.createInvite(identity, relayDeviceId) + const first = await store.reserveCredential(identity.relayHostId, invite.inviteToken, 'first') + expect(first).toMatchObject({ credentialKind: 'invite', reservationId: 'first' }) + expect(await store.reserveCredential(identity.relayHostId, invite.inviteToken, 'second')).toBeNull() + now = first!.leaseExpiresAt + expect(await store.reserveCredential(identity.relayHostId, invite.inviteToken, 'second')).toBeNull() + await database.query(`UPDATE relay_invites SET cooldown_until = ? WHERE token_hash = ?`, [ + now, + hashCredential(invite.inviteToken) + ]) + expect(await store.reserveCredential(identity.relayHostId, invite.inviteToken, 'second')).toMatchObject({ + reservationId: 'second' + }) + await database.close() + }) + + it('validates director moves without reservation and enforces account-global mint rate', async () => { + const database = await openInMemoryRelayDatabase() + const store = new RelayCredentialStore(database, () => 100) + const invite = await store.createInvite(identity, relayDeviceId) + expect(await store.validateInviteForMove(identity.relayHostId, invite.inviteToken)).toBe(true) + const rows = await database.query(`SELECT state, attempt_count FROM relay_invites WHERE token_hash = ?`, [ + hashCredential(invite.inviteToken) + ]) + expect(rows[0]).toMatchObject({ state: 'available', attempt_count: 0 }) + for (let index = 1; index < 30; index++) { + await store.createInvite(identity, `device-${index + 1}`) + } + await expect(store.createInvite(identity, 'device-over-limit')).rejects.toMatchObject({ + code: 'rate_limit_exceeded' + }) + expect(await database.query(`SELECT * FROM relay_audit_events`)).toHaveLength(30) + await database.close() + }) + + it('serializes relay-basis and authenticated-direct under one global result', async () => { + const database = await openInMemoryRelayDatabase() + const store = new RelayCredentialStore(database, () => 100) + const { basisConnId } = await inviteBasis(store) + await store.recordDirectAuthorization({ + ...identity, + relayDeviceId, + directAuthId: 'direct-1', + owningControlGeneration: 1, + deadline: 1_000 + }) + const base = { + ...identity, + relayDeviceId, + reqId: 'req-1', + newResumeTokenHash: hashCredential('pending-resume'), + owningControlGeneration: 1 + } + const [relay, direct] = await Promise.all([ + store.installCredential({ + ...base, + authorization: { mode: 'relay-basis' as const, basisConnId } + }), + store.installCredential({ + ...base, + authorization: { mode: 'authenticated-direct' as const, directAuthId: 'direct-1' } + }) + ]) + expect(relay).toEqual(direct) + expect(relay.currentVersion).toBe(1) + expect(await store.installStatus(base)).toEqual(relay) + const devices = await database.query(`SELECT * FROM relay_devices`) + expect(devices).toHaveLength(1) + await database.close() + }) + + it('rolls back token and invite effects when result persistence fails', async () => { + const database = await openInMemoryRelayDatabase() + const normalStore = new RelayCredentialStore(database, () => 100) + const { basisConnId, reservation } = await inviteBasis(normalStore) + const fault: RelayDatabase = { + query: (sql, params) => database.query(sql, params), + queryLocked: (sql, params) => database.queryLocked(sql, params), + close: () => database.close(), + transaction: async (operation) => + await database.transaction(async (transaction) => + await operation({ + query: async (sql, params) => { + if (sql.includes('INSERT INTO relay_install_results')) throw new Error('injected SQL failure') + return await transaction.query(sql, params) + }, + queryLocked: (sql, params) => transaction.queryLocked(sql, params), + transaction: (nested) => transaction.transaction(nested), + close: () => transaction.close() + }) + ) + } + const faultStore = new RelayCredentialStore(fault, () => 100) + const input = { + ...identity, + relayDeviceId, + reqId: 'req-fault', + newResumeTokenHash: hashCredential('pending-resume'), + owningControlGeneration: 1, + authorization: { mode: 'relay-basis' as const, basisConnId } + } + await expect(faultStore.installCredential(input)).rejects.toThrow('injected SQL failure') + expect(await normalStore.installStatus(input)).toBeNull() + expect(await database.query(`SELECT * FROM relay_devices`)).toEqual([]) + const invites = await database.query(`SELECT state FROM relay_invites WHERE token_hash = ?`, [ + reservation.tokenHash + ]) + expect(invites[0]?.state).toBe('reserved') + await database.close() + }) + + it('renews only a tuple-bound current credential and replays its committed result', async () => { + let now = 100 + const database = await openInMemoryRelayDatabase() + const store = new RelayCredentialStore(database, () => now) + const { basisConnId } = await inviteBasis(store) + const resumeToken = 'resume-token-1' + await store.installCredential({ + ...identity, + relayDeviceId, + reqId: 'install-1', + newResumeTokenHash: hashCredential(resumeToken), + owningControlGeneration: 1, + authorization: { mode: 'relay-basis', basisConnId } + }) + const reservation = await store.reserveCredential(identity.relayHostId, resumeToken) + if (!reservation) throw new Error('resume did not reserve') + expect(await store.resolveResume(identity.relayHostId, resumeToken)).toEqual({ + userId: identity.userId, + relayDeviceId + }) + await store.recordConnectionBasis({ + ...reservation, + basisConnId: 'resume-basis', + owningControlGeneration: 1, + deadline: 1_000 + }) + now = 200 + const confirmed = await store.confirmResume({ + ...identity, + reqId: 'confirm-1', + basisConnId: 'resume-basis', + owningControlGeneration: 1 + }) + expect(confirmed).toMatchObject({ renewed: true, acceptedAs: 'current' }) + await store.deactivateBasis('resume-basis') + expect( + await store.confirmResume({ + ...identity, + reqId: 'confirm-1', + basisConnId: 'resume-basis', + owningControlGeneration: 1 + }) + ).toEqual(confirmed) + await expect( + store.confirmResume({ + ...identity, + reqId: 'confirm-1', + basisConnId: 'other-basis', + owningControlGeneration: 1 + }) + ).rejects.toMatchObject({ code: 'confirmation_tuple_mismatch' }) + await database.close() + }) + + it('leaves a now-grace version unchanged and rejects late, expired, and revoked tuples', async () => { + let now = 100 + const database = await openInMemoryRelayDatabase() + const store = new RelayCredentialStore(database, () => now) + const { basisConnId } = await inviteBasis(store) + const firstToken = 'resume-token-1' + const first = await store.installCredential({ + ...identity, + relayDeviceId, + reqId: 'install-1', + newResumeTokenHash: hashCredential(firstToken), + owningControlGeneration: 1, + authorization: { mode: 'relay-basis', basisConnId } + }) + const firstReservation = await store.reserveCredential(identity.relayHostId, firstToken) + if (!firstReservation) throw new Error('first resume did not reserve') + await store.recordConnectionBasis({ + ...firstReservation, + basisConnId: 'grace-basis', + owningControlGeneration: 1, + deadline: 100_000_000 + }) + await store.recordDirectAuthorization({ + ...identity, + relayDeviceId, + directAuthId: 'rotate-direct', + owningControlGeneration: 1, + deadline: 1_000 + }) + now = 200 + await store.installCredential({ + ...identity, + relayDeviceId, + reqId: 'install-2', + newResumeTokenHash: hashCredential('resume-token-2'), + expectedCurrentHash: hashCredential(firstToken), + owningControlGeneration: 1, + authorization: { mode: 'authenticated-direct', directAuthId: 'rotate-direct' } + }) + const grace = await store.confirmResume({ + ...identity, + reqId: 'confirm-grace', + basisConnId: 'grace-basis', + owningControlGeneration: 1 + }) + expect(grace).toMatchObject({ acceptedAs: 'current', renewed: false, currentVersion: 2 }) + expect(grace.resumeExpiresAt).toBeGreaterThan(first.resumeExpiresAt) + + now += 24 * 60 * 60 * 1000 + 1 + await expect( + store.confirmResume({ + ...identity, + reqId: 'confirm-expired-grace', + basisConnId: 'grace-basis', + owningControlGeneration: 1 + }) + ).rejects.toMatchObject({ code: 'reject-expired' }) + + const currentReservation = await store.reserveCredential(identity.relayHostId, 'resume-token-2') + if (!currentReservation) throw new Error('current resume did not reserve') + await store.recordConnectionBasis({ + ...currentReservation, + basisConnId: 'revoked-basis', + owningControlGeneration: 1, + deadline: now + 1_000 + }) + await store.revoke(identity, relayDeviceId) + await expect( + store.confirmResume({ + ...identity, + reqId: 'confirm-revoked', + basisConnId: 'revoked-basis', + owningControlGeneration: 1 + }) + ).rejects.toMatchObject({ code: 'reject-revoked' }) + + await store.recordConnectionBasis({ + ...currentReservation, + basisConnId: 'late-basis', + owningControlGeneration: 1, + deadline: now - 1 + }) + await expect( + store.confirmResume({ + ...identity, + reqId: 'confirm-late', + basisConnId: 'late-basis', + owningControlGeneration: 1 + }) + ).rejects.toMatchObject({ code: 'confirmation_not_active' }) + await database.close() + }) +}) diff --git a/cloud/apps/relay/src/credential-store.ts b/cloud/apps/relay/src/credential-store.ts new file mode 100644 index 00000000000..a5697e8c699 --- /dev/null +++ b/cloud/apps/relay/src/credential-store.ts @@ -0,0 +1,745 @@ +import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto' +import { + decideResumeCommit, + RELAY_PROTOCOL_LIMITS, + type DeviceCredentialInstalled, + type DeviceResumeConfirmed +} from '@orca-cloud/relay-contract' +import type { RelayDatabase, SqlRow } from './database.js' + +const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000 +// Released desktops validate invite expiry against their own clock with zero +// tolerance at exactly inviteTtlMs; issuing under the ceiling keeps pairing +// working for clients whose clocks trail the cell by up to this margin. +const INVITE_ISSUE_SKEW_MARGIN_MS = 30 * 1000 + +export type RelayIdentity = { userId: string; relayHostId: string } +export type CredentialReservation = RelayIdentity & { + credentialKind: 'invite' | 'resume' + relayDeviceId: string + tokenHash: string + reservationId: string + leaseExpiresAt: number + acceptedCredentialVersion?: number + acceptedAs?: 'current' | 'grace' + resumeExpiresAt?: number + graceExpiresAt?: number +} + +export type InstallInput = RelayIdentity & { + relayDeviceId: string + reqId: string + newResumeTokenHash: string + expectedCurrentHash?: string + owningControlGeneration: number + authorization: + | { mode: 'relay-basis'; basisConnId: string } + | { mode: 'authenticated-direct'; directAuthId: string } +} + +export class RelayStoreError extends Error { + constructor(readonly code: string) { + super(code) + } +} + +export function hashCredential(token: string): string { + return createHash('sha256').update(token).digest('base64url') +} + +function equalHash(left: string, right: string): boolean { + const leftBytes = Buffer.from(left) + const rightBytes = Buffer.from(right) + return leftBytes.length === rightBytes.length && timingSafeEqual(leftBytes, rightBytes) +} + +function number(row: SqlRow, field: string): number { + const value = Number(row[field]) + if (!Number.isSafeInteger(value)) throw new RelayStoreError(`invalid_${field}`) + return value +} + +function optionalNumber(row: SqlRow, field: string): number | undefined { + return row[field] === null || row[field] === undefined ? undefined : number(row, field) +} + +function string(row: SqlRow, field: string): string { + const value = row[field] + if (typeof value !== 'string') throw new RelayStoreError(`invalid_${field}`) + return value +} + +export class RelayCredentialStore { + constructor( + private readonly database: RelayDatabase, + private readonly now: () => number = Date.now + ) {} + + async createInvite(identity: RelayIdentity, relayDeviceId: string): Promise<{ + inviteToken: string + expiresAt: number + maxAttempts: number + }> { + const inviteToken = randomBytes(32).toString('base64url') + const tokenHash = hashCredential(inviteToken) + const now = this.now() + const expiresAt = now + RELAY_PROTOCOL_LIMITS.inviteTtlMs - INVITE_ISSUE_SKEW_MARGIN_MS + await this.database.transaction(async (transaction) => { + await this.consumeRateWith(transaction, `account:${identity.userId}`, 'invite-mint', 30, 60_000, now) + await transaction.query( + `UPDATE relay_invites SET state = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ? + AND state IN (?, ?, ?)`, + [ + 'invalidated', now, identity.userId, identity.relayHostId, relayDeviceId, + 'available', 'reserved', 'cooldown' + ] + ) + await this.auditWith(transaction, { + type: 'invite-created', + ...identity, + relayDeviceId, + detail: { expiresAt } + }) + await transaction.query( + `INSERT INTO relay_invites + (user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count, + max_attempts, expires_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, identity.relayHostId, relayDeviceId, tokenHash, 'available', 0, + RELAY_PROTOCOL_LIMITS.inviteMaxAttempts, expiresAt, now, now + ] + ) + }) + return { inviteToken, expiresAt, maxAttempts: RELAY_PROTOCOL_LIMITS.inviteMaxAttempts } + } + + async reserveCredential( + relayHostId: string, + token: string, + reservationId: string = randomUUID() + ): Promise { + const tokenHash = hashCredential(token) + const now = this.now() + return await this.database.transaction(async (transaction) => { + const inviteRows = await transaction.query( + `SELECT * FROM relay_invites WHERE relay_host_id = ? AND token_hash = ?`, + [relayHostId, tokenHash] + ) + const invite = inviteRows[0] + if (invite && equalHash(string(invite, 'token_hash'), tokenHash)) { + const expiresAt = number(invite, 'expires_at') + const attempts = number(invite, 'attempt_count') + const maxAttempts = number(invite, 'max_attempts') + let state = string(invite, 'state') + const reservationExpiresAt = optionalNumber(invite, 'reservation_expires_at') + if (expiresAt <= now) state = 'expired' + else if (state === 'reserved' && (reservationExpiresAt ?? 0) <= now) { + await transaction.query( + `UPDATE relay_invites SET state = ?, reservation_id = NULL, + reservation_expires_at = NULL, cooldown_until = ?, updated_at = ? + WHERE token_hash = ?`, + [ + 'cooldown', + now + RELAY_PROTOCOL_LIMITS.inviteAttemptCooldownMs, + now, + tokenHash + ] + ) + return null + } + const cooldownUntil = optionalNumber(invite, 'cooldown_until') ?? 0 + if ( + (state !== 'available' && !(state === 'cooldown' && cooldownUntil <= now)) || + attempts >= maxAttempts + ) { + return null + } + const leaseExpiresAt = Math.min( + expiresAt, + now + RELAY_PROTOCOL_LIMITS.inviteReservationLeaseMs + ) + await transaction.query( + `UPDATE relay_invites SET state = ?, attempt_count = ?, reservation_id = ?, + reservation_expires_at = ?, cooldown_until = NULL, updated_at = ? + WHERE token_hash = ?`, + ['reserved', attempts + 1, reservationId, leaseExpiresAt, now, tokenHash] + ) + return { + userId: string(invite, 'user_id'), + relayHostId, + relayDeviceId: string(invite, 'relay_device_id'), + credentialKind: 'invite', + tokenHash, + reservationId, + leaseExpiresAt + } + } + + const deviceRows = await transaction.query( + `SELECT * FROM relay_devices + WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)`, + [relayHostId, tokenHash, tokenHash] + ) + const device = deviceRows[0] + if (!device || optionalNumber(device, 'revoked_at') !== undefined) return null + const currentHash = string(device, 'current_hash') + const currentExpiresAt = number(device, 'current_expires_at') + const graceHash = device.grace_hash + const graceExpiresAt = optionalNumber(device, 'grace_expires_at') + let acceptedAs: 'current' | 'grace' + let acceptedCredentialVersion: number + if (equalHash(currentHash, tokenHash) && currentExpiresAt > now) { + acceptedAs = 'current' + acceptedCredentialVersion = number(device, 'current_version') + } else if ( + typeof graceHash === 'string' && + equalHash(graceHash, tokenHash) && + (graceExpiresAt ?? 0) > now + ) { + acceptedAs = 'grace' + acceptedCredentialVersion = number(device, 'grace_version') + } else { + return null + } + return { + userId: string(device, 'user_id'), + relayHostId, + relayDeviceId: string(device, 'relay_device_id'), + credentialKind: 'resume', + tokenHash, + reservationId, + leaseExpiresAt: now + RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs, + acceptedCredentialVersion, + acceptedAs, + resumeExpiresAt: currentExpiresAt, + graceExpiresAt + } + }) + } + + async recordConnectionBasis(input: CredentialReservation & { + basisConnId: string + owningControlGeneration: number + deadline: number + }): Promise { + const now = this.now() + await this.database.query( + `INSERT INTO relay_connection_bases + (basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation, + credential_kind, invite_token_hash, accepted_credential_version, accepted_as, + deadline, active, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + input.basisConnId, input.userId, input.relayHostId, input.relayDeviceId, + input.owningControlGeneration, input.credentialKind, + input.credentialKind === 'invite' ? input.tokenHash : null, + input.acceptedCredentialVersion, input.acceptedAs, input.deadline, 1, now + ] + ) + } + + async failReservation(reservation: CredentialReservation): Promise { + if (reservation.credentialKind !== 'invite') return + const now = this.now() + await this.database.transaction(async (transaction) => { + const rows = await transaction.query( + `SELECT attempt_count, max_attempts FROM relay_invites + WHERE token_hash = ? AND reservation_id = ? AND state = ?`, + [reservation.tokenHash, reservation.reservationId, 'reserved'] + ) + const invite = rows[0] + if (!invite) return + const exhausted = number(invite, 'attempt_count') >= number(invite, 'max_attempts') + await transaction.query( + `UPDATE relay_invites SET state = ?, reservation_id = NULL, + reservation_expires_at = NULL, cooldown_until = ?, updated_at = ? + WHERE token_hash = ? AND reservation_id = ?`, + [ + exhausted ? 'invalidated' : 'cooldown', + exhausted ? null : now + RELAY_PROTOCOL_LIMITS.inviteAttemptCooldownMs, + now, + reservation.tokenHash, + reservation.reservationId + ] + ) + }) + } + + async recordDirectAuthorization(input: RelayIdentity & { + relayDeviceId: string + directAuthId: string + owningControlGeneration: number + deadline: number + }): Promise { + await this.database.query( + `INSERT INTO relay_direct_authorizations + (direct_auth_id, user_id, relay_host_id, relay_device_id, + owning_control_generation, deadline) + VALUES (?, ?, ?, ?, ?, ?)`, + [ + input.directAuthId, input.userId, input.relayHostId, input.relayDeviceId, + input.owningControlGeneration, input.deadline + ] + ) + } + + async installCredential(input: InstallInput): Promise { + let lastError: unknown + for (let attempt = 0; attempt <= 3; attempt++) { + try { + return await this.installCredentialOnce(input) + } catch (error) { + if (error instanceof RelayStoreError) throw error + const committed = await this.installStatus(input) + if (committed) return committed + const code = (error as { code?: unknown }).code + const retryable = ['23505', '40P01', '55P03', '40001'].includes(String(code)) + if (!retryable || attempt === 3) throw error + lastError = error + } + } + throw lastError + } + + private async installCredentialOnce(input: InstallInput): Promise { + return await this.database.transaction(async (transaction) => { + const existing = await this.installStatusWith(transaction, input) + if (existing) return existing + const now = this.now() + const basisInviteHash = await this.validateInstallAuthorization(transaction, input, now) + const devices = await transaction.query( + `SELECT * FROM relay_devices + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`, + [input.userId, input.relayHostId, input.relayDeviceId] + ) + const current = devices[0] + if (input.expectedCurrentHash) { + if (!current || !equalHash(string(current, 'current_hash'), input.expectedCurrentHash)) { + throw new RelayStoreError('current_hash_mismatch') + } + } + const currentVersion = current ? number(current, 'current_version') + 1 : 1 + const resumeExpiresAt = now + RELAY_PROTOCOL_LIMITS.resumeTtlMs + const graceExpiresAt = current ? now + CREDENTIAL_GRACE_MS : undefined + await transaction.query( + `INSERT INTO relay_devices + (user_id, relay_host_id, relay_device_id, current_hash, current_version, + current_expires_at, grace_hash, grace_version, grace_expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (user_id, relay_host_id, relay_device_id) DO UPDATE SET + current_hash = excluded.current_hash, + current_version = excluded.current_version, + current_expires_at = excluded.current_expires_at, + grace_hash = excluded.grace_hash, + grace_version = excluded.grace_version, + grace_expires_at = excluded.grace_expires_at, + revoked_at = NULL, + updated_at = excluded.updated_at`, + [ + input.userId, input.relayHostId, input.relayDeviceId, input.newResumeTokenHash, + currentVersion, resumeExpiresAt, current ? string(current, 'current_hash') : null, + current ? number(current, 'current_version') : null, graceExpiresAt, now + ] + ) + if (basisInviteHash) { + await transaction.query( + `UPDATE relay_invites SET state = ?, reservation_id = NULL, + reservation_expires_at = NULL, updated_at = ? WHERE token_hash = ?`, + ['consumed', now, basisInviteHash] + ) + } else { + await transaction.query( + `UPDATE relay_invites SET state = ?, reservation_id = NULL, + reservation_expires_at = NULL, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ? + AND state IN (?, ?, ?)`, + [ + 'invalidated', now, input.userId, input.relayHostId, input.relayDeviceId, + 'available', 'reserved', 'cooldown' + ] + ) + } + await transaction.query( + `UPDATE relay_direct_authorizations SET consumed_at = ? + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ? + AND consumed_at IS NULL`, + [now, input.userId, input.relayHostId, input.relayDeviceId] + ) + const result: DeviceCredentialInstalled = { + v: 1, + reqId: input.reqId, + authorizationMode: input.authorization.mode, + currentVersion, + resumeExpiresAt, + ...(graceExpiresAt === undefined ? {} : { graceExpiresAt }) + } + await transaction.query( + `INSERT INTO relay_install_results + (user_id, relay_host_id, relay_device_id, req_id, authorization_mode, + result_json, committed_at) VALUES (?, ?, ?, ?, ?, ?, ?)`, + [ + input.userId, input.relayHostId, input.relayDeviceId, input.reqId, + input.authorization.mode, JSON.stringify(result), now + ] + ) + await this.auditWith(transaction, { + type: 'credential-installed', + userId: input.userId, + relayHostId: input.relayHostId, + relayDeviceId: input.relayDeviceId, + detail: { reqId: input.reqId, authorizationMode: input.authorization.mode, currentVersion } + }) + return result + }) + } + + async installStatus(input: RelayIdentity & { + relayDeviceId: string + reqId: string + }): Promise { + return await this.installStatusWith(this.database, input) + } + + async confirmResume(input: RelayIdentity & { + reqId: string + basisConnId: string + owningControlGeneration: number + }): Promise { + return await this.database.transaction(async (transaction) => { + const prior = await transaction.query( + `SELECT basis_conn_id, result_json FROM relay_confirm_results + WHERE user_id = ? AND relay_host_id = ? AND req_id = ?`, + [input.userId, input.relayHostId, input.reqId] + ) + if (prior[0]) { + if (string(prior[0], 'basis_conn_id') !== input.basisConnId) { + throw new RelayStoreError('confirmation_tuple_mismatch') + } + return JSON.parse(string(prior[0], 'result_json')) as DeviceResumeConfirmed + } + const rows = await transaction.query( + `SELECT * FROM relay_connection_bases WHERE basis_conn_id = ?`, + [input.basisConnId] + ) + const basis = rows[0] + const now = this.now() + if ( + !basis || + string(basis, 'user_id') !== input.userId || + string(basis, 'relay_host_id') !== input.relayHostId || + string(basis, 'credential_kind') !== 'resume' || + number(basis, 'owning_control_generation') !== input.owningControlGeneration || + number(basis, 'active') !== 1 || + number(basis, 'deadline') < now + ) { + throw new RelayStoreError('confirmation_not_active') + } + const relayDeviceId = string(basis, 'relay_device_id') + await transaction.query( + `UPDATE relay_devices SET updated_at = updated_at + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`, + [input.userId, input.relayHostId, relayDeviceId] + ) + const devices = await transaction.query( + `SELECT * FROM relay_devices + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`, + [input.userId, input.relayHostId, relayDeviceId] + ) + const device = devices[0] + if (!device) throw new RelayStoreError('credential_not_found') + const acceptedVersion = number(basis, 'accepted_credential_version') + const decision = decideResumeCommit( + { + currentVersion: number(device, 'current_version'), + currentHash: string(device, 'current_hash'), + currentExpiresAt: number(device, 'current_expires_at'), + graceVersion: optionalNumber(device, 'grace_version'), + graceHash: typeof device.grace_hash === 'string' ? device.grace_hash : undefined, + graceExpiresAt: optionalNumber(device, 'grace_expires_at'), + revokedAt: optionalNumber(device, 'revoked_at') + }, + acceptedVersion, + now + ) + if (decision.startsWith('reject-')) throw new RelayStoreError(decision) + const renewed = decision === 'renew-current' + const resumeExpiresAt = renewed + ? now + RELAY_PROTOCOL_LIMITS.resumeTtlMs + : number(device, 'current_expires_at') + if (renewed) { + await transaction.query( + `UPDATE relay_devices SET current_expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`, + [resumeExpiresAt, now, input.userId, input.relayHostId, relayDeviceId] + ) + } + const result: DeviceResumeConfirmed = { + v: 1, + reqId: input.reqId, + currentVersion: number(device, 'current_version'), + acceptedAs: string(basis, 'accepted_as') as 'current' | 'grace', + renewed, + resumeExpiresAt, + ...(optionalNumber(device, 'grace_expires_at') === undefined + ? {} + : { graceExpiresAt: optionalNumber(device, 'grace_expires_at') }) + } + await transaction.query( + `INSERT INTO relay_confirm_results + (user_id, relay_host_id, req_id, basis_conn_id, tuple_json, result_json, committed_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [ + input.userId, input.relayHostId, input.reqId, input.basisConnId, + JSON.stringify({ + relayDeviceId, + acceptedCredentialVersion: acceptedVersion, + acceptedAs: result.acceptedAs, + confirmDeadline: number(basis, 'deadline'), + owningControlGeneration: input.owningControlGeneration + }), + JSON.stringify(result), + now + ] + ) + await this.auditWith(transaction, { + type: 'resume-confirmed', + userId: input.userId, + relayHostId: input.relayHostId, + relayDeviceId, + detail: { reqId: input.reqId, basisConnId: input.basisConnId, renewed } + }) + return result + }) + } + + async deactivateBasis(basisConnId: string): Promise { + await this.database.query(`UPDATE relay_connection_bases SET active = ? WHERE basis_conn_id = ?`, [ + 0, + basisConnId + ]) + } + + async revoke(identity: RelayIdentity, relayDeviceId: string): Promise { + await this.database.transaction(async (transaction) => { + const now = this.now() + await transaction.query( + `UPDATE relay_devices SET revoked_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`, + [now, now, identity.userId, identity.relayHostId, relayDeviceId] + ) + await transaction.query( + `UPDATE relay_invites SET state = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`, + ['invalidated', now, identity.userId, identity.relayHostId, relayDeviceId] + ) + await this.auditWith(transaction, { + type: 'device-revoked', + ...identity, + relayDeviceId, + detail: {} + }) + }) + } + + async resolveResume( + relayHostId: string, + token: string + ): Promise<{ userId: string; relayDeviceId: string } | null> { + const tokenHash = hashCredential(token) + const rows = await this.database.query( + `SELECT * FROM relay_devices + WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)`, + [relayHostId, tokenHash, tokenHash] + ) + const device = rows[0] + if (!device || optionalNumber(device, 'revoked_at') !== undefined) return null + const now = this.now() + const currentValid = + equalHash(string(device, 'current_hash'), tokenHash) && + number(device, 'current_expires_at') > now + const graceHash = device.grace_hash + const graceValid = + typeof graceHash === 'string' && + equalHash(graceHash, tokenHash) && + (optionalNumber(device, 'grace_expires_at') ?? 0) > now + return currentValid || graceValid + ? { userId: string(device, 'user_id'), relayDeviceId: string(device, 'relay_device_id') } + : null + } + + async validateInviteForMove(relayHostId: string, token: string): Promise { + return Boolean(await this.resolveInviteForMove(relayHostId, token)) + } + + async resolveInviteForMove( + relayHostId: string, + token: string + ): Promise<{ userId: string; relayDeviceId: string } | null> { + const tokenHash = hashCredential(token) + const rows = await this.database.query( + `SELECT user_id, relay_device_id, token_hash, state, attempt_count, max_attempts, expires_at + FROM relay_invites WHERE relay_host_id = ? AND token_hash = ?`, + [relayHostId, tokenHash] + ) + const invite = rows[0] + const valid = Boolean( + invite && + equalHash(string(invite, 'token_hash'), tokenHash) && + ['available', 'reserved', 'cooldown'].includes(string(invite, 'state')) && + number(invite, 'attempt_count') < number(invite, 'max_attempts') && + number(invite, 'expires_at') > this.now() + ) + return valid && invite + ? { userId: string(invite, 'user_id'), relayDeviceId: string(invite, 'relay_device_id') } + : null + } + + async consumeRate(input: { + scopeKey: string + kind: string + limit: number + windowMs: number + }): Promise { + await this.database.transaction(async (transaction) => { + await this.consumeRateWith( + transaction, + input.scopeKey, + input.kind, + input.limit, + input.windowMs, + this.now() + ) + }) + } + + async cleanup(): Promise { + const now = this.now() + await this.database.transaction(async (transaction) => { + await transaction.query( + `UPDATE relay_invites SET state = ?, reservation_id = NULL, + reservation_expires_at = NULL, updated_at = ? + WHERE expires_at <= ? AND state IN (?, ?, ?)`, + ['expired', now, now, 'available', 'reserved', 'cooldown'] + ) + await transaction.query( + `UPDATE relay_invites SET state = ?, reservation_id = NULL, + reservation_expires_at = NULL, cooldown_until = ?, updated_at = ? + WHERE state = ? AND reservation_expires_at <= ? AND expires_at > ?`, + ['cooldown', now + RELAY_PROTOCOL_LIMITS.inviteAttemptCooldownMs, now, 'reserved', now, now] + ) + await transaction.query( + `UPDATE relay_connection_bases SET active = ? WHERE active = ? AND deadline <= ?`, + [0, 1, now] + ) + await transaction.query( + `UPDATE relay_direct_authorizations SET consumed_at = ? + WHERE consumed_at IS NULL AND deadline <= ?`, + [now, now] + ) + await transaction.query( + `DELETE FROM relay_rate_windows WHERE window_started_at < ?`, + [now - 24 * 60 * 60 * 1000] + ) + }) + } + + private async installStatusWith( + database: RelayDatabase, + input: RelayIdentity & { relayDeviceId: string; reqId: string } + ): Promise { + const rows = await database.query( + `SELECT result_json FROM relay_install_results + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ? AND req_id = ?`, + [input.userId, input.relayHostId, input.relayDeviceId, input.reqId] + ) + return rows[0] ? (JSON.parse(string(rows[0], 'result_json')) as DeviceCredentialInstalled) : null + } + + private async validateInstallAuthorization( + transaction: RelayDatabase, + input: InstallInput, + now: number + ): Promise { + if (input.authorization.mode === 'relay-basis') { + const rows = await transaction.query( + `SELECT * FROM relay_connection_bases WHERE basis_conn_id = ?`, + [input.authorization.basisConnId] + ) + const basis = rows[0] + if ( + !basis || + string(basis, 'user_id') !== input.userId || + string(basis, 'relay_host_id') !== input.relayHostId || + string(basis, 'relay_device_id') !== input.relayDeviceId || + string(basis, 'credential_kind') !== 'invite' || + number(basis, 'owning_control_generation') !== input.owningControlGeneration || + number(basis, 'active') !== 1 || + number(basis, 'deadline') < now + ) { + throw new RelayStoreError('invalid_relay_basis') + } + return string(basis, 'invite_token_hash') + } + const rows = await transaction.query( + `SELECT * FROM relay_direct_authorizations WHERE direct_auth_id = ?`, + [input.authorization.directAuthId] + ) + const direct = rows[0] + if ( + !direct || + string(direct, 'user_id') !== input.userId || + string(direct, 'relay_host_id') !== input.relayHostId || + string(direct, 'relay_device_id') !== input.relayDeviceId || + number(direct, 'owning_control_generation') !== input.owningControlGeneration || + number(direct, 'deadline') < now || + optionalNumber(direct, 'consumed_at') !== undefined + ) { + throw new RelayStoreError('invalid_direct_authorization') + } + await transaction.query( + `UPDATE relay_direct_authorizations SET consumed_at = ? WHERE direct_auth_id = ?`, + [now, input.authorization.directAuthId] + ) + return null + } + + private async consumeRateWith( + transaction: RelayDatabase, + scopeKey: string, + kind: string, + limit: number, + windowMs: number, + now: number + ): Promise { + const windowStartedAt = Math.floor(now / windowMs) * windowMs + const rows = await transaction.query( + `INSERT INTO relay_rate_windows + (scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?) + ON CONFLICT (scope_key, window_kind, window_started_at) DO UPDATE + SET count = relay_rate_windows.count + 1 RETURNING count`, + [scopeKey, kind, windowStartedAt, 1] + ) + if (number(rows[0]!, 'count') > limit) throw new RelayStoreError('rate_limit_exceeded') + } + + private async auditWith( + transaction: RelayDatabase, + event: RelayIdentity & { + type: string + relayDeviceId?: string + detail: Record + } + ): Promise { + await transaction.query( + `INSERT INTO relay_audit_events + (id, at, type, user_id, relay_host_id, relay_device_id, detail_json) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [ + randomUUID(), this.now(), event.type, event.userId, event.relayHostId, + event.relayDeviceId, JSON.stringify(event.detail) + ] + ) + } +} diff --git a/cloud/apps/relay/src/database-postgres-timeout.test.ts b/cloud/apps/relay/src/database-postgres-timeout.test.ts new file mode 100644 index 00000000000..a04a9eea042 --- /dev/null +++ b/cloud/apps/relay/src/database-postgres-timeout.test.ts @@ -0,0 +1,201 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const fakes = vi.hoisted(() => ({ + configs: [] as Array>, + query: vi.fn(async () => ({ rows: [], rowCount: 0 })), + release: vi.fn(), + end: vi.fn(async () => undefined) +})) + +vi.mock('pg', () => ({ + default: { + Pool: class { + totalCount = 1 + idleCount = 1 + waitingCount = 0 + end = fakes.end + on = vi.fn() + connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release })) + + constructor(config: Record) { + fakes.configs.push(config) + } + } + } +})) + +import { openRelayDatabase } from './database.js' +import { applyPostgresSchema } from './postgres-schema-startup.js' + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('PostgreSQL relay deadlines', () => { + beforeEach(() => { + fakes.configs.length = 0 + fakes.query.mockClear() + fakes.release.mockClear() + fakes.end.mockClear() + }) + + it('bounds pool acquisition, statements, locks, and abandoned transactions', async () => { + const database = await openRelayDatabase({ + databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay', + dataDir: './unused', + poolMax: 3, + applicationName: 'orca-relay/director/director' + }) + + expect(fakes.configs).toEqual([ + expect.objectContaining({ + max: 3, + application_name: 'orca-relay/director/director', + connectionTimeoutMillis: 2_000, + statement_timeout: 5_000, + lock_timeout: 1_000, + idle_in_transaction_session_timeout: 5_000 + }) + ]) + await database.close() + }) +}) + +describe('PostgreSQL schema startup', () => { + it('retries lock and statement timeouts with bounded backoff', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const query = vi + .fn<(statement: string) => Promise>() + .mockRejectedValueOnce(Object.assign(new Error('lock timeout'), { code: '55P03' })) + .mockRejectedValueOnce(Object.assign(new Error('statement timeout'), { code: '57014' })) + .mockResolvedValue(undefined) + const delays: number[] = [] + + await applyPostgresSchema(['CREATE TABLE test'], query, { + random: () => 0, + wait: async (delayMs) => { + delays.push(delayMs) + } + }) + + expect(query).toHaveBeenCalledTimes(3) + expect(delays).toEqual([125, 250]) + }) + + it('retries only the PostgreSQL concurrent type-creation collision', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const collision = Object.assign(new Error('duplicate type'), { + code: '23505', + constraint: 'pg_type_typname_nsp_index' + }) + const query = vi + .fn<(statement: string) => Promise>() + .mockRejectedValueOnce(collision) + .mockResolvedValue(undefined) + + await applyPostgresSchema(['CREATE TABLE IF NOT EXISTS test'], query, { + wait: async () => undefined + }) + + expect(query).toHaveBeenCalledTimes(2) + }) + + it('retries only the PostgreSQL concurrent index-creation collision', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const collision = Object.assign(new Error('duplicate index'), { + code: '23505', + constraint: 'pg_class_relname_nsp_index' + }) + const query = vi + .fn<(statement: string) => Promise>() + .mockRejectedValueOnce(collision) + .mockResolvedValue(undefined) + + await applyPostgresSchema(['CREATE INDEX IF NOT EXISTS test_index ON test(id)'], query, { + wait: async () => undefined + }) + + expect(query).toHaveBeenCalledTimes(2) + }) + + it.each([ + ['pg_type_typname_nsp_index', 'CREATE TABLE test'], + ['pg_class_relname_nsp_index', 'CREATE INDEX test_index ON test(id)'] + ])('does not retry %s for non-idempotent DDL', async (constraint, statement) => { + const error = Object.assign(new Error('duplicate catalog object'), { + code: '23505', + constraint + }) + const query = vi.fn<(statement: string) => Promise>().mockRejectedValue(error) + const pause = vi.fn(async () => undefined) + + await expect( + applyPostgresSchema([statement], query, { wait: pause }) + ).rejects.toBe(error) + + expect(pause).not.toHaveBeenCalled() + }) + + it('does not retry unrelated unique violations', async () => { + const error = Object.assign(new Error('duplicate row'), { + code: '23505', + constraint: 'application_key' + }) + const query = vi.fn<(statement: string) => Promise>().mockRejectedValue(error) + const pause = vi.fn(async () => undefined) + + await expect( + applyPostgresSchema(['CREATE TABLE test'], query, { wait: pause }) + ).rejects.toBe(error) + + expect(pause).not.toHaveBeenCalled() + }) + + it('fails immediately for non-timeout schema errors', async () => { + const error = Object.assign(new Error('permission denied'), { code: '42501' }) + const query = vi.fn<(statement: string) => Promise>().mockRejectedValue(error) + const pause = vi.fn(async () => undefined) + + await expect( + applyPostgresSchema(['CREATE TABLE test'], query, { wait: pause }) + ).rejects.toBe(error) + + expect(query).toHaveBeenCalledTimes(1) + expect(pause).not.toHaveBeenCalled() + }) + + it('stops retrying at the shared startup deadline', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const error = Object.assign(new Error('lock timeout'), { code: '55P03' }) + const delays: number[] = [] + let now = 0 + const query = vi + .fn<(statement: string) => Promise>() + .mockImplementationOnce(async () => { + now = 200 + }) + .mockRejectedValue(error) + + await expect( + applyPostgresSchema(['CREATE TABLE first', 'CREATE TABLE second'], query, { + now: () => now, + random: () => 1, + retryDeadlineMs: 300, + wait: async (delayMs) => { + delays.push(delayMs) + now += delayMs + } + }) + ).rejects.toBe(error) + + expect(query).toHaveBeenCalledTimes(3) + expect(delays).toEqual([100]) + expect(console.warn).toHaveBeenLastCalledWith( + JSON.stringify({ + event: 'orca_relay_postgres_schema_retry_exhausted', + code: '55P03', + attempts: 2 + }) + ) + }) +}) diff --git a/cloud/apps/relay/src/database-transient-error.test.ts b/cloud/apps/relay/src/database-transient-error.test.ts new file mode 100644 index 00000000000..b29ff519328 --- /dev/null +++ b/cloud/apps/relay/src/database-transient-error.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest' +import { isRelayDatabaseTransientError } from './database.js' + +describe('relay database transient errors', () => { + it.each(['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'])( + 'classifies PostgreSQL code %s as retryable overload', + (code) => { + expect(isRelayDatabaseTransientError({ code })).toBe(true) + } + ) + + it('classifies pool acquisition timeout without hiding programming failures', () => { + expect( + isRelayDatabaseTransientError(new Error('timeout exceeded when trying to connect')) + ).toBe(true) + expect(isRelayDatabaseTransientError(new TypeError('broken invariant'))).toBe(false) + }) +}) diff --git a/cloud/apps/relay/src/database.test.ts b/cloud/apps/relay/src/database.test.ts new file mode 100644 index 00000000000..32e50a7bc6a --- /dev/null +++ b/cloud/apps/relay/src/database.test.ts @@ -0,0 +1,154 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { openInMemoryRelayDatabase, openRelayDatabase } from './database.js' + +const temporaryDirectories: string[] = [] + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('relay database', () => { + it('creates every durable relay state table', async () => { + const database = await openInMemoryRelayDatabase() + const rows = await database.query( + `SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'relay_%' ORDER BY name` + ) + expect(rows.map((row) => row.name)).toEqual([ + 'relay_admission_selector_cell_additions', + 'relay_admission_selector_intents', + 'relay_admission_selectors', + 'relay_assignment_activity_leases', + 'relay_assignment_migration_incarnations', + 'relay_assignment_migrations', + 'relay_assignment_region_preferences', + 'relay_assignments', + 'relay_audit_events', + 'relay_cell_admission', + 'relay_cell_capabilities', + 'relay_cell_committed_fences', + 'relay_cell_connection_limits', + 'relay_cell_connection_runtime', + 'relay_cell_connection_snapshots', + 'relay_cell_drain_attempt_states', + 'relay_cell_drain_attempts', + 'relay_cell_drain_recovery_attempts', + 'relay_cell_fence_apply_invocations', + 'relay_cell_fence_attempts', + 'relay_cell_fence_plan_bindings', + 'relay_cell_fences', + 'relay_cell_legacy_fence_adoptions', + 'relay_cell_regions', + 'relay_cell_rehome_safety', + 'relay_cell_runtime', + 'relay_cells', + 'relay_confirm_results', + 'relay_confirmable_splices', + 'relay_connection_bases', + 'relay_control_connection_reservations', + 'relay_devices', + 'relay_direct_authorizations', + 'relay_install_results', + 'relay_invites', + 'relay_migration_leases', + 'relay_post_drain_migration_pins', + 'relay_rate_windows', + 'relay_region_rehome_attempts', + 'relay_region_rehome_control', + 'relay_region_rehome_worker_state' + ]) + await database.close() + }) + + it('rolls back every effect and serializes concurrent transactions', async () => { + const database = await openInMemoryRelayDatabase() + await expect( + database.transaction(async (transaction) => { + await transaction.query( + `INSERT INTO relay_rate_windows + (scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?)`, + ['scope', 'invite', 1, 1] + ) + throw new Error('injected failure') + }) + ).rejects.toThrow('injected failure') + expect(await database.query(`SELECT * FROM relay_rate_windows`)).toEqual([]) + + await Promise.all([ + database.transaction(async (transaction) => { + await transaction.query( + `INSERT INTO relay_rate_windows + (scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?)`, + ['scope', 'invite', 1, 1] + ) + }), + database.transaction(async (transaction) => { + const rows = await transaction.query( + `SELECT count FROM relay_rate_windows + WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`, + ['scope', 'invite', 1] + ) + await transaction.query( + `UPDATE relay_rate_windows SET count = ? + WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`, + [Number(rows[0]?.count ?? 0) + 1, 'scope', 'invite', 1] + ) + }) + ]) + const rows = await database.query(`SELECT count FROM relay_rate_windows`) + expect(Number(rows[0]?.count)).toBe(2) + await database.close() + }) + + it('persists SQLite state across process-style reopen', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-db-')) + temporaryDirectories.push(dataDir) + const first = await openRelayDatabase({ dataDir }) + await first.query( + `INSERT INTO relay_rate_windows + (scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?)`, + ['scope', 'connect', 1, 7] + ) + await first.close() + const second = await openRelayDatabase({ dataDir }) + const rows = await second.query(`SELECT count FROM relay_rate_windows`) + expect(Number(rows[0]?.count)).toBe(7) + await second.close() + }) + + it('defaults cells created by an older schema user to the US on reopen', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-region-db-')) + temporaryDirectories.push(dataDir) + const first = await openRelayDatabase({ dataDir }) + await first.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES (?, ?, 1, 10, 0, 0, 1, 1)`, + ['legacy-cell', 'https://legacy.relay.example.com'] + ) + await first.close() + + const second = await openRelayDatabase({ dataDir }) + expect( + await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [ + 'legacy-cell' + ]) + ).toEqual([{ region: 'us-central1' }]) + await second.close() + }) + + it('indexes region preference expiry by observation time', async () => { + const database = await openInMemoryRelayDatabase() + const rows = await database.query( + `SELECT sql FROM sqlite_master + WHERE type = 'index' AND name = 'relay_assignment_region_preferences_observed'` + ) + expect(rows[0]?.sql).toContain('(observed_at)') + await database.close() + }) +}) diff --git a/cloud/apps/relay/src/database.ts b/cloud/apps/relay/src/database.ts new file mode 100644 index 00000000000..f7208863f42 --- /dev/null +++ b/cloud/apps/relay/src/database.ts @@ -0,0 +1,935 @@ +import { mkdirSync } from 'node:fs' +import { join } from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import pg from 'pg' +import { + emptyPostgresPoolPressureCounts, + PostgresPoolPressure, + type PostgresPoolPressureCounts +} from './postgres-pool-pressure.js' +import { applyPostgresSchema } from './postgres-schema-startup.js' + +export type SqlRow = Record +export type RelayLockOptions = { failIfUnavailable?: boolean } +export type RelayTransactionOptions = { reportRetries?: boolean } + +export interface RelayDatabase { + readonly dialect?: 'sqlite' | 'postgres' + query(sql: string, params?: unknown[]): Promise + queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise + transaction( + operation: (transaction: RelayDatabase) => Promise, + options?: RelayTransactionOptions + ): Promise + close(): Promise +} + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS relay_invites ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + relay_device_id TEXT NOT NULL, + token_hash TEXT NOT NULL UNIQUE, + state TEXT NOT NULL, + attempt_count BIGINT NOT NULL, + max_attempts BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + reservation_id TEXT, + reservation_expires_at BIGINT, + cooldown_until BIGINT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, relay_device_id, token_hash) +); +CREATE INDEX IF NOT EXISTS relay_invites_device + ON relay_invites(user_id, relay_host_id, relay_device_id); + +CREATE TABLE IF NOT EXISTS relay_devices ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + relay_device_id TEXT NOT NULL, + current_hash TEXT NOT NULL, + current_version BIGINT NOT NULL, + current_expires_at BIGINT NOT NULL, + grace_hash TEXT, + grace_version BIGINT, + grace_expires_at BIGINT, + revoked_at BIGINT, + updated_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, relay_device_id) +); +CREATE INDEX IF NOT EXISTS relay_devices_current_hash ON relay_devices(relay_host_id, current_hash); +CREATE INDEX IF NOT EXISTS relay_devices_grace_hash ON relay_devices(relay_host_id, grace_hash); + +CREATE TABLE IF NOT EXISTS relay_install_results ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + relay_device_id TEXT NOT NULL, + req_id TEXT NOT NULL, + authorization_mode TEXT NOT NULL, + result_json TEXT NOT NULL, + committed_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, relay_device_id, req_id) +); + +CREATE TABLE IF NOT EXISTS relay_confirmable_splices ( + basis_conn_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + owning_control_generation BIGINT NOT NULL, + relay_device_id TEXT NOT NULL, + accepted_credential_version BIGINT NOT NULL, + accepted_as TEXT NOT NULL, + confirm_deadline BIGINT NOT NULL, + active BIGINT NOT NULL, + created_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_connection_bases ( + basis_conn_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + relay_device_id TEXT NOT NULL, + owning_control_generation BIGINT NOT NULL, + credential_kind TEXT NOT NULL, + invite_token_hash TEXT, + accepted_credential_version BIGINT, + accepted_as TEXT, + deadline BIGINT NOT NULL, + active BIGINT NOT NULL, + created_at BIGINT NOT NULL +); + +-- Why: the maintenance sweep matches (active, deadline) while inactive bases +-- accumulate unboundedly. Unindexed it seq-scans millions of rows every cycle +-- and holds the maintenance transaction open long enough to time out +-- assignment lock waits. +CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline + ON relay_connection_bases(active, deadline); + +CREATE TABLE IF NOT EXISTS relay_direct_authorizations ( + direct_auth_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + relay_device_id TEXT NOT NULL, + owning_control_generation BIGINT NOT NULL, + deadline BIGINT NOT NULL, + consumed_at BIGINT +); + +CREATE TABLE IF NOT EXISTS relay_confirm_results ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + req_id TEXT NOT NULL, + basis_conn_id TEXT NOT NULL, + tuple_json TEXT NOT NULL, + result_json TEXT NOT NULL, + committed_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, req_id) +); + +CREATE TABLE IF NOT EXISTS relay_assignments ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + cell_id TEXT NOT NULL, + assignment_epoch BIGINT NOT NULL, + lease_expires_at BIGINT NOT NULL, + last_activity_at BIGINT NOT NULL, + reserved_controls BIGINT NOT NULL, + reserved_splices BIGINT NOT NULL, + reserved_invites BIGINT NOT NULL, + pending_installs BIGINT NOT NULL, + pending_confirmations BIGINT NOT NULL, + migration_leases BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id) +); + +CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + preferred_region TEXT NOT NULL + CHECK (preferred_region IN ('us-central1', 'asia-east2')), + observed_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id) +); +CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed + ON relay_assignment_region_preferences(observed_at); + +CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state ( + worker_id TEXT PRIMARY KEY, + next_dispatch_at BIGINT NOT NULL, + paused_until BIGINT NOT NULL, + consecutive_failures BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_region_rehome_control ( + control_id TEXT PRIMARY KEY, + generation BIGINT NOT NULL, + enabled BIGINT NOT NULL, + observation_started_at BIGINT NOT NULL, + not_before BIGINT NOT NULL, + rate_per_minute BIGINT NOT NULL, + preference_max_age_ms BIGINT NOT NULL, + drain_grace_ms BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts ( + attempt_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'), + source_cell_id TEXT NOT NULL, + source_cell_incarnation TEXT NOT NULL, + target_cell_id TEXT NOT NULL, + target_cell_incarnation TEXT NOT NULL, + previous_epoch BIGINT NOT NULL, + assignment_epoch BIGINT NOT NULL, + drain_grace_ms BIGINT NOT NULL, + send_attempts BIGINT NOT NULL, + last_send_attempt_at BIGINT, + drain_receipt_at BIGINT, + drain_outcome TEXT CHECK ( + drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected') + ), + completed_at BIGINT, + aborted_at BIGINT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + UNIQUE (user_id, relay_host_id, assignment_epoch) +); +CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending + ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at); + +CREATE TABLE IF NOT EXISTS relay_cells ( + cell_id TEXT PRIMARY KEY, + cell_url TEXT NOT NULL UNIQUE, + enabled BIGINT NOT NULL, + capacity_requests BIGINT NOT NULL, + reserved_requests BIGINT NOT NULL, + observed_requests BIGINT NOT NULL, + last_heartbeat_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_cell_regions ( + cell_id TEXT PRIMARY KEY, + region TEXT NOT NULL CHECK (region IN ('us-central1', 'asia-east2')) +); + +CREATE TABLE IF NOT EXISTS relay_cell_admission ( + cell_id TEXT PRIMARY KEY, + admission_state TEXT NOT NULL + CHECK (admission_state IN ('existing-only', 'migration-only', 'general')), + updated_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_admission_selectors ( + selector_id TEXT PRIMARY KEY, + generation BIGINT NOT NULL, + attempt_id TEXT, + membership_json TEXT NOT NULL, + updated_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_admission_selector_intents ( + attempt_id TEXT PRIMARY KEY, + expected_generation BIGINT NOT NULL, + intended_generation BIGINT NOT NULL, + previous_membership_json TEXT NOT NULL, + membership_json TEXT NOT NULL, + created_at BIGINT NOT NULL, + committed_at BIGINT +); + +CREATE TABLE IF NOT EXISTS relay_admission_selector_cell_additions ( + attempt_id TEXT PRIMARY KEY, + cells_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_cell_runtime ( + cell_id TEXT PRIMARY KEY, + cell_url TEXT NOT NULL, + cell_incarnation TEXT NOT NULL, + started_at BIGINT NOT NULL, + ready BIGINT NOT NULL, + observed_requests BIGINT NOT NULL, + last_heartbeat_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_cell_runtime_heartbeat + ON relay_cell_runtime(ready, last_heartbeat_at); + +CREATE TABLE IF NOT EXISTS relay_cell_capabilities ( + cell_id TEXT PRIMARY KEY, + cell_incarnation TEXT NOT NULL, + regional_rehome_protocol BIGINT NOT NULL, + last_heartbeat_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_cell_rehome_safety ( + cell_id TEXT PRIMARY KEY, + cell_incarnation TEXT NOT NULL, + observed_at BIGINT NOT NULL, + sql_failures BIGINT NOT NULL, + reconnects BIGINT NOT NULL, + control_activity_recovery_failures BIGINT NOT NULL, + database_pool_waiting BIGINT NOT NULL, + database_pool_waiters_max BIGINT NOT NULL, + database_pool_wait_ms_max BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_cell_connection_limits ( + cell_id TEXT PRIMARY KEY, + hard_cap BIGINT NOT NULL, + unobserved_bound BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS relay_cell_connection_runtime ( + cell_id TEXT PRIMARY KEY, + cell_incarnation TEXT NOT NULL, + total_connections BIGINT NOT NULL, + in_flight_connections BIGINT NOT NULL, + reserved_connection_units BIGINT NOT NULL, + enforced_connection_units BIGINT NOT NULL, + last_heartbeat_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_cell_connection_runtime_heartbeat + ON relay_cell_connection_runtime(last_heartbeat_at); + +CREATE TABLE IF NOT EXISTS relay_cell_connection_snapshots ( + cell_id TEXT PRIMARY KEY, + cell_incarnation TEXT NOT NULL, + inclusion_watermark BIGINT NOT NULL, + total_connections BIGINT NOT NULL, + in_flight_connections BIGINT NOT NULL, + reserved_connection_units BIGINT NOT NULL, + enforced_connection_units BIGINT NOT NULL, + snapshot_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_cell_connection_snapshot_freshness + ON relay_cell_connection_snapshots(snapshot_at); + +CREATE TABLE IF NOT EXISTS relay_cell_fences ( + cell_id TEXT PRIMARY KEY, + cell_incarnation TEXT NOT NULL, + attested_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_cell_fences_expiry + ON relay_cell_fences(expires_at); + +CREATE TABLE IF NOT EXISTS relay_cell_committed_fences ( + cell_id TEXT PRIMARY KEY, + attempt_id TEXT NOT NULL UNIQUE, + cell_incarnation TEXT NOT NULL, + attested_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_cell_committed_fences_expiry + ON relay_cell_committed_fences(expires_at); + +CREATE TABLE IF NOT EXISTS relay_cell_legacy_fence_adoptions ( + cell_id TEXT PRIMARY KEY, + cell_incarnation TEXT NOT NULL, + attested_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_cell_legacy_fence_adoptions_expiry + ON relay_cell_legacy_fence_adoptions(expires_at); + +CREATE TABLE IF NOT EXISTS relay_cell_fence_attempts ( + attempt_id TEXT PRIMARY KEY, + environment TEXT NOT NULL, + cell_id TEXT NOT NULL, + cell_incarnation TEXT NOT NULL, + mig_name TEXT NOT NULL, + instance_group TEXT NOT NULL, + generation_identity TEXT NOT NULL, + fence_commit TEXT NOT NULL, + plan_sha256 TEXT NOT NULL, + gce_operation TEXT, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + apply_started_at BIGINT, + completed_at BIGINT, + aborted_at BIGINT +); +CREATE INDEX IF NOT EXISTS relay_cell_fence_attempts_expiry + ON relay_cell_fence_attempts(expires_at); +CREATE INDEX IF NOT EXISTS relay_cell_fence_attempts_cell + ON relay_cell_fence_attempts(cell_id, created_at); + +CREATE TABLE IF NOT EXISTS relay_cell_fence_plan_bindings ( + attempt_id TEXT PRIMARY KEY, + plan_object_name TEXT NOT NULL, + plan_object_generation TEXT, + var_file_sha256 TEXT NOT NULL, + terraform_state_lineage TEXT NOT NULL, + terraform_state_serial BIGINT NOT NULL, + terraform_state_object_generation TEXT NOT NULL, + terraform_state_object_sha256 TEXT NOT NULL, + request_reason TEXT NOT NULL, + FOREIGN KEY (attempt_id) REFERENCES relay_cell_fence_attempts(attempt_id) +); + +CREATE TABLE IF NOT EXISTS relay_cell_fence_apply_invocations ( + invocation_id TEXT PRIMARY KEY, + attempt_id TEXT NOT NULL, + request_reason TEXT NOT NULL UNIQUE, + started_at BIGINT NOT NULL, + gce_operation TEXT, + FOREIGN KEY (attempt_id) REFERENCES relay_cell_fence_attempts(attempt_id) +); +CREATE INDEX IF NOT EXISTS relay_cell_fence_apply_invocations_attempt + ON relay_cell_fence_apply_invocations(attempt_id, started_at); + +CREATE TABLE IF NOT EXISTS relay_cell_drain_attempts ( + cell_id TEXT PRIMARY KEY, + cell_incarnation TEXT NOT NULL, + planned_grace_ms BIGINT NOT NULL, + attempted_at BIGINT NOT NULL, + retry_after BIGINT NOT NULL, + recover_forward_attempted_at BIGINT +); + +CREATE TABLE IF NOT EXISTS relay_cell_drain_attempt_states ( + attempt_id TEXT PRIMARY KEY, + cell_id TEXT NOT NULL, + cell_incarnation TEXT NOT NULL, + trace_value TEXT NOT NULL UNIQUE, + planned_grace_ms BIGINT NOT NULL, + state TEXT NOT NULL CHECK ( + state IN ( + 'prepared', + 'send-may-have-started', + 'application-receipt', + 'proven-not-delivered' + ) + ), + prepared_at BIGINT NOT NULL, + send_may_have_started_at BIGINT, + send_permit_expires_at BIGINT, + application_receipt_at BIGINT, + backend_success_status BIGINT, + backend_instance TEXT, + receipt_cell_incarnation TEXT, + retry_after BIGINT, + recover_forward_attempted_at BIGINT, + proven_not_delivered_at BIGINT +); +CREATE INDEX IF NOT EXISTS relay_cell_drain_attempt_states_cell + ON relay_cell_drain_attempt_states(cell_id, prepared_at); + +CREATE TABLE IF NOT EXISTS relay_cell_drain_recovery_attempts ( + drain_attempt_id TEXT NOT NULL, + cell_incarnation TEXT NOT NULL, + attempted_at BIGINT NOT NULL, + PRIMARY KEY (drain_attempt_id, cell_incarnation), + FOREIGN KEY (drain_attempt_id) REFERENCES relay_cell_drain_attempt_states(attempt_id) +); + +CREATE TABLE IF NOT EXISTS relay_assignment_activity_leases ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + activity_id TEXT NOT NULL, + activity_kind TEXT NOT NULL, + cell_id TEXT NOT NULL, + request_units BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, activity_id) +); +CREATE INDEX IF NOT EXISTS relay_assignment_activity_expiry + ON relay_assignment_activity_leases(expires_at); + +CREATE TABLE IF NOT EXISTS relay_control_connection_reservations ( + reservation_id TEXT PRIMARY KEY, + idempotency_key TEXT NOT NULL UNIQUE, + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + assignment_epoch BIGINT NOT NULL, + cell_id TEXT NOT NULL, + state TEXT NOT NULL + CHECK (state IN ('reserved', 'late-arrival-debt', 'claimed', 'released')), + inclusion_watermark BIGINT, + claim_activity_id TEXT, + created_at BIGINT NOT NULL, + timeout_at BIGINT NOT NULL, + claimed_at BIGINT, + released_at BIGINT, + updated_at BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_control_connection_reservation_headroom + ON relay_control_connection_reservations(cell_id, state); +CREATE INDEX IF NOT EXISTS relay_control_connection_reservation_assignment + ON relay_control_connection_reservations( + user_id, relay_host_id, assignment_epoch, cell_id, created_at + ); + +CREATE TABLE IF NOT EXISTS relay_rate_windows ( + scope_key TEXT NOT NULL, + window_kind TEXT NOT NULL, + window_started_at BIGINT NOT NULL, + count BIGINT NOT NULL, + PRIMARY KEY (scope_key, window_kind, window_started_at) +); + +CREATE TABLE IF NOT EXISTS relay_migration_leases ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + source_cell_id TEXT NOT NULL, + target_cell_id TEXT NOT NULL, + assignment_epoch BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + completed_at BIGINT, + PRIMARY KEY (user_id, relay_host_id, assignment_epoch) +); + +CREATE TABLE IF NOT EXISTS relay_assignment_migrations ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + source_cell_id TEXT NOT NULL, + target_cell_id TEXT NOT NULL, + previous_epoch BIGINT NOT NULL, + assignment_epoch BIGINT NOT NULL, + source_request_units BIGINT NOT NULL, + target_reserved_units BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + target_registered_at BIGINT, + completed_at BIGINT, + aborted_at BIGINT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, assignment_epoch) +); +CREATE INDEX IF NOT EXISTS relay_assignment_migrations_active + ON relay_assignment_migrations(expires_at, completed_at, aborted_at); + +CREATE TABLE IF NOT EXISTS relay_assignment_migration_incarnations ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + assignment_epoch BIGINT NOT NULL, + source_cell_incarnation TEXT NOT NULL, + target_cell_incarnation TEXT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, assignment_epoch) +); + +CREATE TABLE IF NOT EXISTS relay_post_drain_migration_pins ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + assignment_epoch BIGINT NOT NULL, + drain_attempt_id TEXT NOT NULL, + source_cell_id TEXT NOT NULL, + source_cell_incarnation TEXT NOT NULL, + target_cell_id TEXT NOT NULL, + target_cell_incarnation TEXT NOT NULL, + source_request_units BIGINT NOT NULL, + target_reserved_units BIGINT NOT NULL, + pinned_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id, assignment_epoch) +); +CREATE INDEX IF NOT EXISTS relay_post_drain_migration_pins_attempt + ON relay_post_drain_migration_pins(drain_attempt_id); + +CREATE TABLE IF NOT EXISTS relay_audit_events ( + id TEXT PRIMARY KEY, + at BIGINT NOT NULL, + type TEXT NOT NULL, + user_id TEXT, + relay_host_id TEXT, + relay_device_id TEXT, + detail_json TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at); +` + +function postgresSql(sql: string): string { + let index = 0 + return sql.replace(/\?/g, () => `$${++index}`) +} + +function returnsRows(sql: string): boolean { + return /^\s*(select|with)/i.test(sql) || /returning/i.test(sql) +} + +const POSTGRES_TRANSACTION_PHASES = [ + ['relay_region_rehome_', 'regional-rehome'], + ['relay_assignment_activity_leases', 'activity-lease'], + ['relay_assignment_migration', 'migration'], + ['relay_migration_leases', 'migration'], + ['relay_post_drain_migration_pins', 'migration'], + ['relay_cell_connection_runtime', 'cell-runtime'], + ['relay_cell_connection_snapshots', 'cell-runtime'], + ['relay_cell_runtime', 'cell-runtime'], + ['relay_cell_drain_', 'cell-operation'], + ['relay_cell_fence', 'cell-operation'], + ['relay_cell_committed_fences', 'cell-operation'], + ['relay_cell_legacy_fence_adoptions', 'cell-operation'], + ['relay_admission_selector', 'admission'], + ['relay_cell_admission', 'admission'], + ['relay_control_connection_reservations', 'connection'], + ['relay_confirmable_splices', 'connection'], + ['relay_connection_bases', 'connection'], + ['relay_direct_authorizations', 'connection'], + ['relay_confirm_results', 'connection'], + ['relay_assignment_region_preferences', 'assignment'], + ['relay_assignments', 'assignment'], + ['relay_cell_', 'cell-inventory'], + ['relay_cells', 'cell-inventory'], + ['relay_invites', 'credential'], + ['relay_devices', 'credential'], + ['relay_install_results', 'credential'], + ['relay_rate_windows', 'rate-limit'], + ['relay_audit_events', 'audit'] +] as const + +const postgresTransactionPhaseByError = new WeakMap() + +function postgresTransactionPhase(sql: string): string { + const normalized = sql.toLowerCase() + return POSTGRES_TRANSACTION_PHASES.find(([table]) => normalized.includes(table))?.[1] ?? 'other' +} + +function rememberPostgresTransactionPhase(error: unknown, sql: string): void { + if (typeof error === 'object' && error !== null) { + postgresTransactionPhaseByError.set(error, postgresTransactionPhase(sql)) + } +} + +function postgresTransactionErrorPhase(error: unknown): string { + return typeof error === 'object' && error !== null + ? (postgresTransactionPhaseByError.get(error) ?? 'transaction') + : 'transaction' +} + +class SqliteTransaction implements RelayDatabase { + readonly dialect = 'sqlite' as const + + constructor(protected readonly database: DatabaseSync) {} + + async query(sql: string, params: unknown[] = []): Promise { + const statement = this.database.prepare(sql) + const bound = params.map((value) => (value === undefined ? null : value)) as never[] + if (returnsRows(sql)) return statement.all(...bound) as SqlRow[] + const result = statement.run(...bound) + return [{ changes: Number(result.changes) }] + } + + async queryLocked( + sql: string, + params: unknown[] = [], + _options: RelayLockOptions = {} + ): Promise { + return await this.query(sql, params) + } + + async transaction( + operation: (transaction: RelayDatabase) => Promise, + _options: RelayTransactionOptions = {} + ): Promise { + return await operation(this) + } + + async close(): Promise {} +} + +class SqliteDatabase extends SqliteTransaction { + private tail: Promise = Promise.resolve() + + override async query(sql: string, params: unknown[] = []): Promise { + await this.tail + return await super.query(sql, params) + } + + override async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + const previous = this.tail + let release!: () => void + this.tail = new Promise((resolve) => (release = resolve)) + await previous + this.database.exec('BEGIN IMMEDIATE') + try { + const result = await operation(new SqliteTransaction(this.database)) + this.database.exec('COMMIT') + return result + } catch (error) { + this.database.exec('ROLLBACK') + throw error + } finally { + release() + } + } + + override async close(): Promise { + await this.tail + this.database.close() + } +} + +class PostgresTransaction implements RelayDatabase { + readonly dialect = 'postgres' as const + + constructor(protected readonly client: pg.PoolClient) {} + + async query(sql: string, params: unknown[] = []): Promise { + try { + const result = await this.client.query(postgresSql(sql), params) + return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }] + } catch (error) { + rememberPostgresTransactionPhase(error, sql) + throw error + } + } + + async queryLocked( + sql: string, + params: unknown[] = [], + options: RelayLockOptions = {} + ): Promise { + try { + return await this.query( + `${sql} FOR UPDATE${options.failIfUnavailable ? ' NOWAIT' : ''}`, + params + ) + } catch (error) { + if ( + options.failIfUnavailable && + String((error as { code?: unknown }).code) === '55P03' + ) { + throw new Error('database_lock_unavailable') + } + throw error + } + } + + async transaction( + operation: (transaction: RelayDatabase) => Promise, + _options: RelayTransactionOptions = {} + ): Promise { + return await operation(this) + } + + async close(): Promise {} +} + +const POSTGRES_TRANSACTION_ATTEMPTS = 3 +const POSTGRES_RETRY_MAX_DELAY_MS = 25 +const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000 +const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000 +const POSTGRES_LOCK_TIMEOUT_MS = 1_000 +const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000 + +function retryablePostgresTransactionError(error: unknown): boolean { + const code = String((error as { code?: unknown }).code) + return code === '40P01' || code === '40001' || code === '55P03' +} + +export function isRelayDatabaseTransientError(error: unknown): boolean { + const code = String((error as { code?: unknown }).code) + if (['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'].includes(code)) { + return true + } + return String((error as { message?: unknown }).message).includes( + 'timeout exceeded when trying to connect' + ) +} + +async function waitForPostgresRetry(random: () => number = Math.random): Promise { + const delayMs = Math.floor(random() * (POSTGRES_RETRY_MAX_DELAY_MS + 1)) + await new Promise((resolve) => setTimeout(resolve, delayMs)) +} + +class PostgresDatabase implements RelayDatabase { + readonly dialect = 'postgres' as const + private readonly pressure: PostgresPoolPressure + + constructor(private readonly pool: pg.Pool) { + this.pressure = new PostgresPoolPressure(pool) + } + + async query(sql: string, params: unknown[] = []): Promise { + const client = await this.pressure.connect() + try { + const result = await client.query(postgresSql(sql), params) + return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }] + } finally { + client.release() + } + } + + async queryLocked( + sql: string, + params: unknown[] = [], + options: RelayLockOptions = {} + ): Promise { + try { + return await this.query( + `${sql} FOR UPDATE${options.failIfUnavailable ? ' NOWAIT' : ''}`, + params + ) + } catch (error) { + if ( + options.failIfUnavailable && + String((error as { code?: unknown }).code) === '55P03' + ) { + throw new Error('database_lock_unavailable') + } + throw error + } + } + + async transaction( + operation: (transaction: RelayDatabase) => Promise, + options: RelayTransactionOptions = {} + ): Promise { + for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) { + const client = await this.pressure.connect() + try { + await client.query('BEGIN') + const result = await operation(new PostgresTransaction(client)) + await client.query('COMMIT') + return result + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined) + if (!retryablePostgresTransactionError(error) || attempt === POSTGRES_TRANSACTION_ATTEMPTS) { + if (retryablePostgresTransactionError(error) && options.reportRetries !== false) { + console.warn( + JSON.stringify({ + event: 'orca_relay_postgres_transaction_exhausted', + code: String((error as { code?: unknown }).code), + attempts: attempt, + phase: postgresTransactionErrorPhase(error) + }) + ) + } + throw error + } + if (options.reportRetries !== false) { + console.warn( + JSON.stringify({ + event: 'orca_relay_postgres_transaction_retry', + code: String((error as { code?: unknown }).code), + attempt, + phase: postgresTransactionErrorPhase(error) + }) + ) + } + } finally { + client.release() + } + // A PostgreSQL transaction is unusable after an abort, so retry all work + // on a fresh pooled client with a small full-jitter delay. + await waitForPostgresRetry() + } + throw new Error('postgres_transaction_retry_exhausted') + } + + async close(): Promise { + await this.pool.end() + } + + consumePoolPressure(): PostgresPoolPressureCounts { + return this.pressure.consumeCounts() + } + + peekPoolPressure(): PostgresPoolPressureCounts { + return this.pressure.peekCounts() + } +} + +export function consumeRelayDatabasePoolPressure( + database: RelayDatabase +): PostgresPoolPressureCounts { + return database instanceof PostgresDatabase + ? database.consumePoolPressure() + : emptyPostgresPoolPressureCounts() +} + +export function readRelayDatabasePoolPressure( + database: RelayDatabase +): PostgresPoolPressureCounts { + return database instanceof PostgresDatabase + ? database.peekPoolPressure() + : emptyPostgresPoolPressureCounts() +} + +export function absorbPostgresIdleClientErrors(pool: Pick): void { + pool.on('error', () => { + // Why: node-postgres removes failed idle clients itself; leaving `error` + // unhandled would crash the cell and turn a SQL outage into autoheal churn. + console.warn('[orca-relay] idle PostgreSQL client failed') + }) +} + +async function applySchema(database: RelayDatabase): Promise { + for (const statement of SCHEMA.split(';')) { + if (statement.trim()) await database.query(statement) + } +} + +async function applySchemaWithPostgresRetries(database: RelayDatabase): Promise { + await applyPostgresSchema( + SCHEMA.split(';').filter((statement) => statement.trim()), + async (statement) => await database.query(statement) + ) +} + +async function backfillRelayCellRegions(database: RelayDatabase): Promise { + await database.query( + `INSERT INTO relay_cell_regions (cell_id, region) + SELECT cell_id, 'us-central1' FROM relay_cells WHERE true + ON CONFLICT (cell_id) DO NOTHING` + ) +} + +export async function openRelayDatabase(input: { + databaseUrl?: string + dataDir: string + poolMax?: number + applicationName?: string +}): Promise { + let database: RelayDatabase + if (input.databaseUrl) { + const pool = new pg.Pool({ + connectionString: input.databaseUrl, + max: input.poolMax ?? 10, + application_name: input.applicationName, + connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS, + statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS, + lock_timeout: POSTGRES_LOCK_TIMEOUT_MS, + idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS + }) + absorbPostgresIdleClientErrors(pool) + database = new PostgresDatabase(pool) + } else { + mkdirSync(input.dataDir, { recursive: true }) + const sqlite = new DatabaseSync(join(input.dataDir, 'orca-relay.sqlite')) + sqlite.exec('PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;') + database = new SqliteDatabase(sqlite) + } + try { + if (input.databaseUrl) await applySchemaWithPostgresRetries(database) + else await applySchema(database) + await backfillRelayCellRegions(database) + return database + } catch (error) { + await database.close().catch(() => undefined) + throw error + } +} + +export async function openInMemoryRelayDatabase(): Promise { + const sqlite = new DatabaseSync(':memory:') + sqlite.exec('PRAGMA foreign_keys = ON;') + const database = new SqliteDatabase(sqlite) + await applySchema(database) + await backfillRelayCellRegions(database) + return database +} diff --git a/cloud/apps/relay/src/fault-injection-test-entry.ts b/cloud/apps/relay/src/fault-injection-test-entry.ts new file mode 100644 index 00000000000..3f2cc6b6246 --- /dev/null +++ b/cloud/apps/relay/src/fault-injection-test-entry.ts @@ -0,0 +1,58 @@ +import { loadRelayConfig } from './config.js' +import { reconcileCellAdmissionAtStartup } from './cell-admission-startup.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' +import { createRelayServer } from './relay-server.js' +import { readFileSync } from 'node:fs' + +if (process.env.NODE_ENV !== 'test') { + throw new Error('fault injection entry is test-only') +} + +const pattern = process.env.ORCA_RELAY_TEST_FAULT_SQL +const clockFile = process.env.ORCA_RELAY_TEST_CLOCK_FILE +if (!pattern && !clockFile) throw new Error('a test fault configuration is required') + +const config = loadRelayConfig() +const realDatabase = await openRelayDatabase({ + databaseUrl: config.databaseUrl, + dataDir: config.dataDir +}) +let faulted = false + +function wrap(transaction: RelayDatabase): RelayDatabase { + return { + query: async (sql, params) => { + if (pattern && !faulted && sql.includes(pattern)) { + faulted = true + throw new Error('injected SQL failure') + } + return await transaction.query(sql, params) + }, + queryLocked: async (sql, params, options) => + await transaction.queryLocked(sql, params, options), + transaction: async (operation) => + await transaction.transaction(async (nested) => await operation(wrap(nested))), + close: async () => await transaction.close() + } +} + +const database = wrap(realDatabase) +const now = clockFile + ? (): number => { + const offset = Number(readFileSync(clockFile, 'utf8')) + if (!Number.isFinite(offset)) throw new Error('invalid test clock offset') + return Date.now() + offset + } + : Date.now +const { server, sessions, assignments } = createRelayServer(config, database, { now }) +await reconcileCellAdmissionAtStartup(config, assignments) +server.listen(config.port, () => { + console.log(`[orca-relay] listening on ${config.publicUrl} (port ${config.port})`) +}) + +const shutdown = (): void => { + sessions.drain(0) + server.close(() => void realDatabase.close()) +} +process.once('SIGTERM', shutdown) +process.once('SIGINT', shutdown) diff --git a/cloud/apps/relay/src/google-metadata-identity-token.ts b/cloud/apps/relay/src/google-metadata-identity-token.ts new file mode 100644 index 00000000000..f51f6282a65 --- /dev/null +++ b/cloud/apps/relay/src/google-metadata-identity-token.ts @@ -0,0 +1,22 @@ +const METADATA_IDENTITY_ENDPOINT = + 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity' +const METADATA_IDENTITY_TIMEOUT_MS = 5_000 + +export async function googleMetadataIdentityToken( + audience: string, + fetchImpl: typeof fetch = fetch +): Promise { + const endpoint = new URL(METADATA_IDENTITY_ENDPOINT) + endpoint.searchParams.set('audience', audience) + endpoint.searchParams.set('format', 'full') + const response = await fetchImpl(endpoint, { + headers: { 'Metadata-Flavor': 'Google' }, + signal: AbortSignal.timeout(METADATA_IDENTITY_TIMEOUT_MS) + }) + if (!response.ok) throw new Error(`metadata_identity_${response.status}`) + const token = (await response.text()).trim() + if (token.length === 0 || token.length > 16 * 1024) { + throw new Error('metadata_identity_invalid') + } + return token +} diff --git a/cloud/apps/relay/src/host-session-registry.test.ts b/cloud/apps/relay/src/host-session-registry.test.ts new file mode 100644 index 00000000000..f632d5d4358 --- /dev/null +++ b/cloud/apps/relay/src/host-session-registry.test.ts @@ -0,0 +1,1007 @@ +import { EventEmitter } from 'node:events' +import { + ASSIGNMENT_LIMITS, + CONTROL_CONTINUITY_LIMITS, + RELAY_CLOSE_CODE, + RELAY_PROTOCOL_LIMITS +} from '@orca-cloud/relay-contract' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type WebSocket from 'ws' +import type { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import type { RelayCredentialStore } from './credential-store.js' +import { HostSessionRegistry, type HostSession } from './host-session-registry.js' +import { relayHostLogDigest } from './relay-host-log-digest.js' +import type { RelayRuntimeObserver } from './relay-observability.js' +import { + REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + REGIONAL_REHOME_TRUST_PROBE_USER_ID +} from './regional-rehome-trust-probe.js' +import type { RelayTokenClaims } from './relay-token-verifier.js' +import { ProcessQueuedByteBudget } from './splice-forwarder.js' + +class FakeSocket extends EventEmitter { + readonly OPEN = 1 + readonly CLOSING = 2 + readonly CLOSED = 3 + readyState = this.OPEN + readonly send = vi.fn() + readonly close = vi.fn((code?: number, reason?: string) => { + this.readyState = this.CLOSED + this.emit('close', code, Buffer.from(reason ?? '')) + }) + readonly terminate = vi.fn(() => { + this.readyState = this.CLOSED + this.emit('close') + }) +} + +type ActivateSession = ( + socket: WebSocket, + identity: RelayTokenClaims, + existing: HostSession | null, + generation: number, + rebind: boolean, + assignmentEpoch: number, + appVersion?: string +) => Promise + +const config = { + port: 8080, + publicUrl: 'https://relay-c3.example.com', + cellUrl: 'https://relay-c3.example.com', + authIssuer: 'https://auth.example.com', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.com/jwks', + assignmentSigningKey: new Uint8Array(32), + role: 'cell', + cellId: 'production-gce-c3', + cells: [ + { + id: 'production-gce-c3', + url: 'https://relay-c3.example.com', + capacityRequests: 4_000 + } + ], + adminAudience: 'https://relay-c3.example.com/v1/admin/drain', + deployServiceAccount: 'deploy@example.com', + runtimeServiceAccount: 'runtime@example.com', + adminJwksUrl: 'https://auth.example.com/admin-jwks', + databasePoolMax: 10, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './test-data' +} satisfies RelayConfig + +const identity = { + sub: 'user-1', + prof: 'profile-1', + relayHostId: 'abcdefghijklmnop', + purpose: 'host-control', + exp: 4_102_444_800 +} satisfies RelayTokenClaims + +function deferred(): { + promise: Promise + resolve(value: T): void +} { + let resolve!: (value: T) => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} + +function createRegistry( + activateControl: RelayAssignmentStore['activateControl'], + store: Partial = {}, + verifyRelayToken: (token: string) => Promise = vi.fn() +): { + registry: HostSessionRegistry + activate: ActivateSession + acquireActivity: ReturnType + renewControlActivity: ReturnType + releaseActivity: ReturnType + observer: { + recordControlClose: ReturnType + recordSpliceClose: ReturnType + } +} { + const acquireActivity = vi.fn().mockResolvedValue(undefined) + const renewControlActivity = vi.fn().mockResolvedValue(undefined) + const releaseActivity = vi.fn().mockResolvedValue(true) + const assignments = { + activateControl, + markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined), + resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }), + acquireActivity, + renewControlActivity, + releaseActivity + } as unknown as RelayAssignmentStore + const observer = { + recordAuth: vi.fn(), + recordForwardedBytes: vi.fn(), + recordHttp: vi.fn(), + recordReconnect: vi.fn(), + recordSql: vi.fn(), + recordControlClose: vi.fn(), + recordSpliceClose: vi.fn() + } satisfies RelayRuntimeObserver + const registry = new HostSessionRegistry( + config, + verifyRelayToken, + store as RelayCredentialStore, + assignments, + new ProcessQueuedByteBudget(), + observer + ) + // Mirrors the production signature exactly so a future positional shift fails to compile. + const bound = ( + registry as unknown as { + activate: ( + socket: WebSocket, + identity: RelayTokenClaims, + existing: HostSession | null, + generation: number, + rebind: boolean, + assignmentEpoch: number, + appVersion: string, + connectionInclusionWatermark?: number + ) => Promise + } + ).activate.bind(registry) + const activate: ActivateSession = ( + socket, + identity, + existing, + generation, + rebind, + assignmentEpoch, + appVersion = '1.4.173' + ) => bound(socket, identity, existing, generation, rebind, assignmentEpoch, appVersion) + return { registry, activate, acquireActivity, renewControlActivity, releaseActivity, observer } +} + +describe('host session cleanup races', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + it('logs control closes with a host digest and counts them, never the raw id', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { activate, observer } = createRegistry(activateControl) + const socket = new FakeSocket() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + socket.emit('error', new RangeError('Max payload size exceeded')) + socket.close(1006, 'network reset') + + expect(observer.recordControlClose).toHaveBeenCalledWith(1006) + const line = warn.mock.calls + .map((call) => String(call[0])) + .find((entry) => entry.includes('control closed')) + expect(line).toContain(`host=${relayHostLogDigest(identity.relayHostId)}`) + expect(line).toContain('code=1006') + expect(line).toContain('Max payload size exceeded') + expect(line).not.toContain(identity.relayHostId) + } finally { + warn.mockRestore() + } + }) + + it('contains a dependency failure to one socket instead of the process', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + // The same rejection shape as a pg-pool connect timeout; unguarded, it + // became an unhandled rejection that crashed whole production cells. + const verifyRelayToken = vi.fn(async (): Promise => { + throw new Error('Connection terminated due to connection timeout') + }) + const { activate } = createRegistry(activateControl, {}, verifyRelayToken) + const socket = new FakeSocket() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + await activate(socket as unknown as WebSocket, identity, null, 1, false, 7) + socket.emit( + 'message', + Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'refreshed' })), + false + ) + await vi.waitFor(() => + expect(socket.close).toHaveBeenCalledWith( + RELAY_CLOSE_CODE.LIMIT_EXCEEDED, + 'relay temporarily unavailable' + ) + ) + expect(warn).toHaveBeenCalledWith( + '[orca-relay] auth refresh failed: Connection terminated due to connection timeout' + ) + } finally { + warn.mockRestore() + } + }) + + it('attributes a control close to its client build without trusting the version string', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { activate } = createRegistry(activateControl) + const socket = new FakeSocket() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + // A client controls this string, so it must be bounded and stripped like any close reason. + await activate( + socket as unknown as WebSocket, + identity, + null, + 1, + false, + 1, + `1.4.173\n${'x'.repeat(200)}` + ) + socket.close(4408, 'replaced by a newer generation') + + const line = warn.mock.calls + .map((call) => String(call[0])) + .find((entry) => entry.includes('control closed')) + expect(line).toContain('app="1.4.173') + // The raw newline is escaped by JSON.stringify either way; only its escaped + // form proves the strip ran, so assert on that. + expect(line).not.toContain('\\n') + expect(line).toMatch(/app="[^"]{1,80}"/) + } finally { + warn.mockRestore() + } + }) + + it('reports the work a generation replacement destroyed, not the drained aftermath', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate } = createRegistry(activateControl, { + failReservation: vi.fn().mockResolvedValue(undefined) + }) + const firstSocket = new FakeSocket() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + await activate(firstSocket as unknown as WebSocket, identity, null, 1, false, 1) + const session = registry.get({ + userId: identity.sub, + relayHostId: identity.relayHostId + })! + // Teardown drains both maps before closing the socket, so a close handler that + // reads them live always reports zero regardless of what was actually killed. + session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a')) + session.activeSplices.set('conn-b', () => session.activeSplices.delete('conn-b')) + const clientSocket = new FakeSocket() + session.pendingConns.set('conn-c', { + connId: 'conn-c', + connTicket: 'ticket', + reservation: { userId: identity.sub, relayHostId: identity.relayHostId }, + client: clientSocket as unknown as WebSocket, + attachTimer: setTimeout(() => {}, 60_000), + credentialActivityId: null + } as unknown as Parameters[1]) + + const secondSocket = new FakeSocket() + await activate(secondSocket as unknown as WebSocket, identity, session, 2, false, 1) + + const line = warn.mock.calls + .map((call) => String(call[0])) + .find((entry) => entry.includes('replaced by a newer generation')) + expect(line).toContain('splices=2') + expect(line).toContain('pending=1') + } finally { + warn.mockRestore() + } + }) + + it('keeps the first drain snapshot when a drain is retried', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate } = createRegistry(activateControl) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + // Captured before draining, because teardown removes the session from the map. + const session = registry.get({ + userId: identity.sub, + relayHostId: identity.relayHostId + })! + session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a')) + + // POST /v1/admin/drain has no idempotency guard, and SIGTERM then SIGINT both + // reach drain(), so a second teardown can be scheduled for the same session. + registry.drain(0) + const scheduled = vi.getTimerCount() + registry.drain(0) + // Pin the premise: if drain ever gains an idempotency guard, the retry schedules no + // second teardown and the assertion below stops defending the write-once snapshot + // while still passing. Compare against the count before the retry rather than an + // absolute, since the session's heartbeat interval is also pending. + expect(vi.getTimerCount()).toBe(scheduled + 1) + vi.advanceTimersByTime(1) + + // Asserting registry state, not the log line: FakeSocket closes synchronously, so + // the line is already emitted before the second teardown runs and would pass either way. + expect(session.closingCounts).toEqual({ splices: 1, pending: 0 }) + }) + + it('drains only the incarnation-bound host and makes replay idempotent', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate } = createRegistry( + activateControl, + {}, + vi.fn(async () => identity) + ) + const firstSocket = new FakeSocket() + const secondSocket = new FakeSocket() + const secondIdentity = { + ...identity, + sub: 'user-2', + relayHostId: 'ponmlkjihgfedcba' + } + await activate(firstSocket as unknown as WebSocket, identity, null, 1, false, 7) + await activate(secondSocket as unknown as WebSocket, secondIdentity, null, 1, false, 3) + const trustProbe = { + attemptId: REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + userId: REGIONAL_REHOME_TRUST_PROBE_USER_ID, + relayHostId: REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + sourceAssignmentEpoch: 1, + graceMs: 0 + } + expect(registry.get(trustProbe)).toBeNull() + expect(registry.drainHost(trustProbe)).toBe('host-not-connected') + expect(registry.drainHost(trustProbe)).toBe('host-not-connected') + expect(firstSocket.send).not.toHaveBeenCalledWith(expect.stringContaining('"drain"')) + expect(secondSocket.send).not.toHaveBeenCalledWith(expect.stringContaining('"drain"')) + const request = { + attemptId: '11111111-1111-4111-8111-111111111111', + userId: identity.sub, + relayHostId: identity.relayHostId, + sourceAssignmentEpoch: 7, + graceMs: 30_000 + } + + expect(registry.drainHost(request)).toBe('accepted') + expect(firstSocket.send).toHaveBeenCalledWith( + JSON.stringify({ type: 'drain', graceMs: 30_000, recovery: 'resolve-director' }) + ) + expect(secondSocket.send).not.toHaveBeenCalledWith(expect.stringContaining('"drain"')) + firstSocket.emit( + 'message', + Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'refreshed' })), + false + ) + await vi.waitFor(() => expect(registry.get(request)?.state).toBe('drain-only')) + const timers = vi.getTimerCount() + expect(registry.drainHost(request)).toBe('already-accepted') + expect(vi.getTimerCount()).toBe(timers) + expect(() => + registry.drainHost({ + ...request, + attemptId: '22222222-2222-4222-8222-222222222222' + }) + ).toThrow('regional_rehome_attempt_conflict') + expect(() => + registry.drainHost({ ...request, sourceAssignmentEpoch: 8 }) + ).toThrow('regional_rehome_assignment_epoch_mismatch') + + const rebound = new FakeSocket() + await activate( + rebound as unknown as WebSocket, + identity, + registry.get(request), + 1, + true, + 7 + ) + expect(registry.get(request)?.state).toBe('drain-only') + expect(rebound.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"')) + + await vi.advanceTimersByTimeAsync(30_000) + expect(registry.get(request)).toBeNull() + expect(registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId })) + .not.toBeNull() + expect(secondSocket.close).not.toHaveBeenCalled() + }) + + it('refreshes the logged client build when a control rebinds', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate } = createRegistry(activateControl) + const firstSocket = new FakeSocket() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + await activate(firstSocket as unknown as WebSocket, identity, null, 1, false, 1, '1.4.100') + const session = registry.get({ + userId: identity.sub, + relayHostId: identity.relayHostId + })! + const rebindSocket = new FakeSocket() + await activate(rebindSocket as unknown as WebSocket, identity, session, 1, true, 1, '1.4.200') + + // The rebind closes the predecessor. That line is a churn line, so it must carry the + // build that socket ran, not the successor's — the refresh above lands before it closes. + const rebound = warn.mock.calls + .map((call) => String(call[0])) + .find((entry) => entry.includes('control rebound')) + expect(rebound).toContain('app="1.4.100"') + + rebindSocket.close(1006, 'network reset') + const line = warn.mock.calls + .map((call) => String(call[0])) + .find((entry) => entry.includes('code=1006')) + expect(line).toContain('app="1.4.200"') + } finally { + warn.mockRestore() + } + }) + + it('keeps every live control socket indexed when first activations overlap', async () => { + const firstControl = deferred() + const secondControl = deferred() + const activateControl = vi + .fn() + .mockReturnValueOnce(firstControl.promise) + .mockReturnValueOnce(secondControl.promise) + const { registry, activate } = createRegistry(activateControl) + const firstSocket = new FakeSocket() + const secondSocket = new FakeSocket() + + const first = activate(firstSocket as unknown as WebSocket, identity, null, 1, false, 1) + const second = activate(secondSocket as unknown as WebSocket, identity, null, 1, false, 1) + secondControl.resolve('control:production-gce-c3:1') + await Promise.resolve() + firstControl.resolve('control:production-gce-c3:1') + await Promise.all([first, second]) + + const liveSockets = [firstSocket, secondSocket].filter( + (socket) => socket.readyState === socket.OPEN + ) + const session = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId }) + expect(liveSockets).toHaveLength(1) + expect(session?.socket).toBe(liveSockets[0]) + }) + + it('does not publish a control that closes during activation', async () => { + const blocked = deferred() + const activateControl = vi + .fn() + .mockReturnValueOnce(blocked.promise) + const { registry, activate, releaseActivity } = createRegistry(activateControl) + const socket = new FakeSocket() + + const activation = activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + socket.close() + blocked.resolve('control:production-gce-c3:1') + await activation + + expect(registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })).toBeNull() + expect(releaseActivity).toHaveBeenCalledWith( + { userId: identity.sub, relayHostId: identity.relayHostId }, + 'control:production-gce-c3:1' + ) + }) + + it('does not rebind a control that closes during activation', async () => { + const blocked = deferred() + const activateControl = vi + .fn() + .mockResolvedValueOnce('control:production-gce-c3:1') + .mockReturnValueOnce(blocked.promise) + const { registry, activate, releaseActivity } = createRegistry(activateControl) + const originalSocket = new FakeSocket() + await activate(originalSocket as unknown as WebSocket, identity, null, 1, false, 1) + const original = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId }) + expect(original).not.toBeNull() + + const rebindSocket = new FakeSocket() + const rebinding = activate( + rebindSocket as unknown as WebSocket, + identity, + original, + 1, + true, + 1 + ) + rebindSocket.close() + blocked.resolve('control:production-gce-c3:1') + await rebinding + + const session = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId }) + expect(session).toBe(original) + expect(session?.socket).toBe(originalSocket) + expect(session?.state).toBe('active') + expect(releaseActivity).toHaveBeenCalledWith( + { userId: identity.sub, relayHostId: identity.relayHostId }, + 'control:production-gce-c3:1' + ) + }) + + it('rejects client lookup when the indexed control socket is not open', async () => { + const reservation = { + userId: identity.sub, + relayHostId: identity.relayHostId, + credentialKind: 'resume', + relayDeviceId: 'device-1', + leaseExpiresAt: Date.now() + 60_000 + } + const store = { + resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }), + reserveCredential: vi.fn().mockResolvedValue(reservation), + failReservation: vi.fn().mockResolvedValue(undefined) + } + const activateControl = vi + .fn() + .mockResolvedValueOnce('control:production-gce-c3:1') + const { registry, activate } = createRegistry(activateControl, store) + const controlSocket = new FakeSocket() + await activate(controlSocket as unknown as WebSocket, identity, null, 1, false, 1) + const session = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId }) + expect(session?.state).toBe('active') + // A dead socket that never delivered its close event: state stays active, + // so only the readyState guard can protect the lookup. + controlSocket.readyState = controlSocket.CLOSED + + const client = new FakeSocket() + await registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'credential') + + expect(store.failReservation).toHaveBeenCalledWith(reservation) + expect(client.send).toHaveBeenCalledWith( + JSON.stringify({ type: 'relay-hello', ok: false, code: RELAY_CLOSE_CODE.HOST_OFFLINE }) + ) + expect(session?.pendingConns.size).toBe(0) + }) + + it('fails a control waiting behind a stalled activation without breaking serialization', async () => { + const stalled = deferred() + const activateControl = vi + .fn() + .mockReturnValueOnce(stalled.promise) + const { registry, activate } = createRegistry(activateControl) + const stalledSocket = new FakeSocket() + const first = activate(stalledSocket as unknown as WebSocket, identity, null, 1, false, 1) + const waitingSocket = new FakeSocket() + const second = activate(waitingSocket as unknown as WebSocket, identity, null, 1, false, 1) + + // Let the first activation reach the store (clearing its own queue timer) + // before the waiting control's deadline elapses. + await Promise.resolve() + await Promise.resolve() + vi.advanceTimersByTime(30_000) + expect(waitingSocket.close).toHaveBeenCalledWith( + RELAY_CLOSE_CODE.LIMIT_EXCEEDED, + 'control activation queue stalled' + ) + // The waiting control never reached the store; serialization held. + expect(activateControl).toHaveBeenCalledOnce() + + stalled.resolve('control:production-gce-c3:1') + await Promise.all([first, second]) + const session = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId }) + expect(session?.socket).toBe(stalledSocket) + expect(session?.state).toBe('active') + }) + + it('rejects an activation that returns after drain begins', async () => { + const blocked = deferred() + const activateControl = vi + .fn() + .mockResolvedValueOnce('control:production-gce-c3:1') + .mockReturnValueOnce(blocked.promise) + const { registry, activate, renewControlActivity, releaseActivity } = + createRegistry(activateControl) + const originalSocket = new FakeSocket() + await activate(originalSocket as unknown as WebSocket, identity, null, 1, false, 1) + const original = registry.get({ + userId: identity.sub, + relayHostId: identity.relayHostId + }) + expect(original).not.toBeNull() + + const replacementSocket = new FakeSocket() + const replacement = activate( + replacementSocket as unknown as WebSocket, + identity, + original, + 2, + false, + 1 + ) + registry.drain(100) + blocked.resolve('control:production-gce-c3:2') + await replacement + vi.advanceTimersByTime(100) + vi.advanceTimersByTime(15_000) + + expect(replacementSocket.close).toHaveBeenCalledWith( + RELAY_CLOSE_CODE.DRAINING, + 'relay draining' + ) + expect(registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })).toBeNull() + expect(renewControlActivity).not.toHaveBeenCalled() + expect(releaseActivity).toHaveBeenCalledWith( + { userId: identity.sub, relayHostId: identity.relayHostId }, + 'control:production-gce-c3:2' + ) + }) + + it('keeps a replacement mapped after stale orphan cleanup', async () => { + const activateControl = vi + .fn() + .mockResolvedValueOnce('control:production-gce-c3:1') + .mockResolvedValueOnce('control:production-gce-c3:2') + const { registry, activate } = createRegistry(activateControl) + const originalSocket = new FakeSocket() + await activate(originalSocket as unknown as WebSocket, identity, null, 1, false, 1) + const original = registry.get({ + userId: identity.sub, + relayHostId: identity.relayHostId + }) + expect(original).not.toBeNull() + originalSocket.close() + + const replacementSocket = new FakeSocket() + await activate( + replacementSocket as unknown as WebSocket, + identity, + original, + 2, + false, + 1 + ) + const replacement = registry.get({ + userId: identity.sub, + relayHostId: identity.relayHostId + }) + vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs) + + expect(replacement).not.toBeNull() + expect(registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })).toBe( + replacement + ) + expect(registry.runtimeCounts().controls).toBe(1) + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('stops the heartbeat of an actively replaced generation', async () => { + const activateControl = vi + .fn() + .mockResolvedValueOnce('control:production-gce-c3:1') + .mockResolvedValueOnce('control:production-gce-c3:2') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + const originalSocket = new FakeSocket() + await activate(originalSocket as unknown as WebSocket, identity, null, 1, false, 1) + const original = registry.get({ + userId: identity.sub, + relayHostId: identity.relayHostId + }) + expect(original).not.toBeNull() + + await activate( + new FakeSocket() as unknown as WebSocket, + identity, + original, + 2, + false, + 1 + ) + vi.advanceTimersByTime(15_000) + + expect(renewControlActivity).toHaveBeenCalledOnce() + expect(renewControlActivity).toHaveBeenCalledWith( + { userId: identity.sub, relayHostId: identity.relayHostId }, + expect.objectContaining({ + activityId: 'control:production-gce-c3:2', + cellId: 'production-gce-c3' + }) + ) + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('keeps 15s pings while halving steady-state control renewals', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + const socket = new FakeSocket() + const activatedAt = Date.now() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + for (let interval = 0; interval < 4; interval++) { + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false) + } + + const pings = socket.send.mock.calls.filter((call) => + String(call[0]).includes('"ping"') + ) + expect(pings).toHaveLength(4) + expect(renewControlActivity).toHaveBeenCalledTimes(2) + const firstExpiry = Number(renewControlActivity.mock.calls[0]![1].expiresAt) + const secondExpiry = Number(renewControlActivity.mock.calls[1]![1].expiresAt) + expect(firstExpiry).toBe( + activatedAt + + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + + ASSIGNMENT_LIMITS.activityLeaseMs + + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + ) + expect(secondExpiry - firstExpiry).toBe(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2) + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('retries a failed control renewal on the next ping', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + renewControlActivity.mockRejectedValueOnce(new Error('pool timeout')) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const socket = new FakeSocket() + try { + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + expect(renewControlActivity).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + expect(renewControlActivity).toHaveBeenCalledTimes(2) + } finally { + warn.mockRestore() + registry.drain(0) + vi.advanceTimersByTime(0) + } + }) + + it('retries past a stalled control renewal without waiting for it', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + const stalled = deferred() + renewControlActivity.mockReturnValueOnce(stalled.promise) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2) + + expect(renewControlActivity).toHaveBeenCalledTimes(2) + stalled.resolve(undefined) + await vi.advanceTimersByTimeAsync(0) + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('ignores a superseded renewal resolving after a fresher success', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + const stalled = deferred() + renewControlActivity.mockReturnValueOnce(stalled.promise) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2) + expect(renewControlActivity).toHaveBeenCalledTimes(2) + stalled.resolve(undefined) + await vi.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + + expect(renewControlActivity).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + + expect(renewControlActivity).toHaveBeenCalledTimes(3) + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('re-acquires a missing control activity lease', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, acquireActivity, renewControlActivity } = + createRegistry(activateControl) + renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found')) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + + expect(acquireActivity).toHaveBeenCalledWith( + { userId: identity.sub, relayHostId: identity.relayHostId }, + { + activityId: 'control:production-gce-c3:1', + kind: 'control', + cellId: config.cellId + } + ) + expect(socket.close).not.toHaveBeenCalled() + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('closes a control whose activity lease moved to another cell', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, acquireActivity, renewControlActivity } = + createRegistry(activateControl) + renewControlActivity.mockRejectedValueOnce(new Error('control_activity_moved')) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + + expect(acquireActivity).not.toHaveBeenCalled() + expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved') + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('closes when a missing control activity cannot be re-acquired on this cell', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, acquireActivity, renewControlActivity } = + createRegistry(activateControl) + renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found')) + acquireActivity.mockRejectedValueOnce(new Error('activity_cell_not_authoritative')) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + + expect(socket.close).toHaveBeenCalledWith( + RELAY_CLOSE_CODE.DRAINING, + 'control migration completed' + ) + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('closes a control when renewal finds its cell is no longer authoritative', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + renewControlActivity.mockRejectedValueOnce(new Error('activity_cell_not_authoritative')) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + + expect(socket.close).toHaveBeenCalledWith( + RELAY_CLOSE_CODE.DRAINING, + 'control migration completed' + ) + registry.drain(0) + vi.advanceTimersByTime(0) + }) + + it('continues renewing throughout the drain grace period', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + registry.drain(60_000) + + for (let interval = 0; interval < 3; interval++) { + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false) + } + + expect(renewControlActivity).toHaveBeenCalledTimes(2) + expect(socket.readyState).toBe(socket.OPEN) + vi.advanceTimersByTime(15_000) + }) +}) + +describe('control renewal cadence across a rebind', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + it('keeps the halved cadence after a rebind lands under a stalled renewal', async () => { + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate, renewControlActivity } = createRegistry(activateControl) + const ping = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + + const beat = async (target: FakeSocket): Promise => { + await vi.advanceTimersByTimeAsync(ping) + target.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false) + } + + // Age the session so its attempt counter is well above zero. + for (let tick = 0; tick < 5; tick++) await beat(socket) + expect(renewControlActivity).toHaveBeenCalledTimes(3) + + // The next renewal stalls and is still in flight when the control rebinds. + const stalled = deferred() + renewControlActivity.mockReturnValueOnce(stalled.promise) + for (let tick = 0; tick < 2; tick++) await beat(socket) + expect(renewControlActivity).toHaveBeenCalledTimes(4) + + const session = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })! + const rebindSocket = new FakeSocket() + await activate(rebindSocket as unknown as WebSocket, identity, session, 1, true, 1) + stalled.resolve(undefined) + await vi.advanceTimersByTimeAsync(0) + + const before = renewControlActivity.mock.calls.length + for (let tick = 0; tick < 4; tick++) await beat(rebindSocket) + expect(renewControlActivity.mock.calls.length - before).toBe(2) + + registry.drain(0) + vi.advanceTimersByTime(0) + }) +}) + +describe('control lease recovery after the session is gone', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + it('does not re-acquire a lease for a session a newer generation already replaced', async () => { + const activateControl = vi + .fn() + .mockResolvedValueOnce('control:production-gce-c3:1') + .mockResolvedValueOnce('control:production-gce-c3:2') + const { registry, activate, acquireActivity, renewControlActivity } = + createRegistry(activateControl) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + try { + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + const session = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })! + + // The renewal is still in flight when a newer generation takes over. + let failRenewal!: (error: Error) => void + renewControlActivity.mockReturnValueOnce( + new Promise((_resolve, reject) => (failRenewal = reject)) + ) + await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs) + expect(renewControlActivity).toHaveBeenCalledOnce() + + const newer = new FakeSocket() + await activate(newer as unknown as WebSocket, identity, session, 2, false, 1) + + // Its release already removed the lease, so the renewal reports it missing. + failRenewal(new Error('control_activity_not_found')) + await vi.advanceTimersByTimeAsync(0) + + expect(acquireActivity).not.toHaveBeenCalled() + } finally { + warn.mockRestore() + registry.drain(0) + vi.advanceTimersByTime(0) + } + }) +}) diff --git a/cloud/apps/relay/src/host-session-registry.ts b/cloud/apps/relay/src/host-session-registry.ts new file mode 100644 index 00000000000..11b7d1de030 --- /dev/null +++ b/cloud/apps/relay/src/host-session-registry.ts @@ -0,0 +1,1226 @@ +import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto' +import { + ASSIGNMENT_LIMITS, + AuthRefreshSchema, + buildHostChallengePlaintext, + buildHostProofMacInput, + buildHostProofTranscript, + CONTROL_CONTINUITY_LIMITS, + DeviceCredentialInstallSchema, + DeviceCredentialInstallStatusSchema, + DeviceResumeConfirmSchema, + DeviceRevokeSchema, + HostChallengeAckSchema, + HostHelloSchema, + InviteCreateSchema, + RELAY_PROTOCOL_LIMITS, + RELAY_CLOSE_CODE +} from '@orca-cloud/relay-contract' +import nacl from 'tweetnacl' +import type WebSocket from 'ws' +import type { RawData } from 'ws' +import type { RelayConfig } from './config.js' +import type { RelayAssignmentStore } from './assignment-store.js' +import { + RelayCredentialStore, + type CredentialReservation +} from './credential-store.js' +import { relayHostLogDigest } from './relay-host-log-digest.js' +import type { RelayTokenClaims } from './relay-token-verifier.js' +import type { RelayRuntimeObserver } from './relay-observability.js' +import type { PendingHostDataReservation } from './relay-connection-ledger.js' +import { closeRelayWebSocket } from './relay-websocket-close.js' +import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js' + +// Peer-supplied close reasons are logged; keep them printable and short. +function printableCloseReason(reason: Buffer | string): string { + return reason + .toString() + .replace(/[^\x20-\x7e]/g, '') + .slice(0, 80) +} + +type VerifyRelayToken = (token: string) => Promise +type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed' + +const CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2 +// Preserve the existing 75s renewal runway after doubling the successful-call interval. +const CONTROL_ACTIVITY_LEASE_MS = + ASSIGNMENT_LIMITS.activityLeaseMs + + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS - + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + +export type HostSession = { + identity: RelayTokenClaims + readonly relayHostId: string + readonly generation: number + readonly assignmentEpoch: number + readonly controlActivityId: string | null + readonly controlResumeSecret: string + // Why: reconnect churn is only actionable once it can be pinned to a client build. + // Refreshed on rebind so it describes the socket that closed, not the first one. + appVersion: string + state: HostState + socket: WebSocket | null + leaseExpiresAt: number + orphanTimer: ReturnType | null + heartbeatTimer: ReturnType | null + lastPongAt: number + activityRenewalDueAt: number + activityRenewalAttempt: number + activityRenewalCompletedAttempt: number + activeConnIds: Set + activeSplices: Map void> + pendingConns: Map + // Why: relay-initiated teardown drains these maps before closing the control + // socket, so the close handler would otherwise always report zero destroyed work. + closingCounts: { splices: number; pending: number } | null + regionalDrainAttemptId: string | null + regionalDrainTimer: ReturnType | null + regionalDrainExpiresAt: number | null +} + +export type RegionalHostDrainOutcome = + | 'accepted' + | 'already-accepted' + | 'host-not-connected' + +type PendingConnection = { + connId: string + connTicket: string + reservation: CredentialReservation + client: WebSocket + attachTimer: ReturnType + credentialActivityId: string | null + capacityReservation?: PendingHostDataReservation +} + +function decodeCanonicalBase64(value: string, bytes: number): Uint8Array | null { + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) { + return null + } + const decoded = Buffer.from(value, 'base64') + return decoded.length === bytes && decoded.toString('base64') === value ? decoded : null +} + +function relayHostId(publicKey: Uint8Array): string { + return createHash('sha256').update(publicKey).digest('base64url').slice(0, 16) +} + +function payload(raw: RawData, expectedType: string): unknown { + if (typeof raw !== 'string' && !Buffer.isBuffer(raw)) return null + try { + const parsed = JSON.parse(raw.toString()) as Record + if (parsed.type !== expectedType) return null + const { type: _type, ...rest } = parsed + return rest + } catch { + return null + } +} + +function send(socket: WebSocket, type: string, message: object): void { + socket.send(JSON.stringify({ type, ...message })) +} + +// Hosts abandon connects after 15s; waiting much longer than that behind a +// stalled predecessor only accumulates doomed sockets. +const ACTIVATION_QUEUE_WAIT_MS = 30_000 + +export class HostSessionRegistry { + private readonly sessions = new Map() + private readonly activationQueues = new Map>() + private draining = false + + constructor( + private readonly config: RelayConfig, + private readonly verifyRelayToken: VerifyRelayToken, + private readonly store: RelayCredentialStore, + private readonly assignments: RelayAssignmentStore, + private readonly queuedByteBudget: ProcessQueuedByteBudget, + private readonly observer: RelayRuntimeObserver, + private readonly now: () => number = Date.now + ) {} + + async acceptClient( + socket: WebSocket, + hostId: string, + credential: string, + capacityReservation?: PendingHostDataReservation + ): Promise { + if (this.draining) { + capacityReservation?.release() + this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING) + return + } + if (this.config.role === 'cell') { + const outerIdentity = + (await this.store.resolveResume(hostId, credential)) ?? + (await this.store.resolveInviteForMove(hostId, credential)) + const assignment = outerIdentity + ? await this.assignments.resolve({ userId: outerIdentity.userId, relayHostId: hostId }) + : null + if (!assignment || assignment.cellId !== this.config.cellId) { + capacityReservation?.release() + this.observer.recordAuth(false) + this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL) + return + } + } + const reservation = await this.store.reserveCredential(hostId, credential) + if (!reservation) { + capacityReservation?.release() + this.observer.recordAuth(false) + this.rejectClient(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL) + return + } + this.observer.recordAuth(true) + const session = this.sessions.get(this.key(reservation.userId, hostId)) + if ( + !session || + session.state !== 'active' || + !session.socket || + session.socket.readyState !== session.socket.OPEN + ) { + capacityReservation?.release() + await this.store.failReservation(reservation) + this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE) + return + } + if (session.activeConnIds.size + session.pendingConns.size >= 8) { + capacityReservation?.release() + await this.store.failReservation(reservation) + this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED) + return + } + const connId = randomUUID() + const connTicket = randomBytes(32).toString('base64url') + const identity = { userId: reservation.userId, relayHostId: hostId } + const credentialActivityId = + this.config.role === 'cell' + ? `${reservation.credentialKind === 'invite' ? 'invite' : 'confirmation'}:${connId}` + : null + if (credentialActivityId) { + try { + await this.assignments.acquireActivity(identity, { + activityId: credentialActivityId, + kind: reservation.credentialKind === 'invite' ? 'invite' : 'confirmation', + cellId: this.config.cellId + }) + } catch { + capacityReservation?.release() + await this.store.failReservation(reservation) + this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED) + return + } + } + const attachTimer = setTimeout(() => { + session.pendingConns.delete(connId) + capacityReservation?.release() + this.failReservationBestEffort(reservation) + if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId) + this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE) + }, RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs) + const pending: PendingConnection = { + connId, + connTicket, + reservation, + client: socket, + attachTimer, + credentialActivityId, + capacityReservation + } + capacityReservation?.bind(connId) + session.pendingConns.set(connId, pending) + send(session.socket, 'conn-open', { + connId, + connTicket, + kind: reservation.credentialKind, + relayDeviceId: reservation.relayDeviceId, + attachDeadlineMs: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs + }) + socket.once('close', () => { + const current = session.pendingConns.get(connId) + if (current?.client === socket) { + clearTimeout(current.attachTimer) + session.pendingConns.delete(connId) + current.capacityReservation?.release() + this.failReservationBestEffort(current.reservation) + if (current.credentialActivityId) { + this.releaseActivityBestEffort(identity, current.credentialActivityId) + } + } + }) + } + + async acceptHostData( + socket: WebSocket, + connId: string, + connTicket: string, + generation: number + ): Promise { + const session = [...this.sessions.values()].find((candidate) => + candidate.pendingConns.has(connId) + ) + const pending = session?.pendingConns.get(connId) + if ( + !session || + !pending || + pending.connTicket !== connTicket || + session.generation !== generation || + (session.state !== 'active' && session.state !== 'drain-only') + ) { + this.observer.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host data ticket') + return false + } + this.observer.recordAuth(true) + clearTimeout(pending.attachTimer) + session.pendingConns.delete(connId) + session.activeConnIds.add(connId) + const basisDeadline = + pending.reservation.credentialKind === 'resume' + ? this.now() + RELAY_PROTOCOL_LIMITS.resumeConfirmationDeadlineMs + : pending.reservation.leaseExpiresAt + const identity = { + userId: pending.reservation.userId, + relayHostId: pending.reservation.relayHostId + } + const spliceActivityId = this.config.role === 'cell' ? `splice:${connId}` : null + try { + if (spliceActivityId) { + await this.assignments.acquireActivity(identity, { + activityId: spliceActivityId, + kind: 'splice', + cellId: this.config.cellId + }) + } + await this.store.recordConnectionBasis({ + ...pending.reservation, + basisConnId: connId, + owningControlGeneration: session.generation, + deadline: basisDeadline + }) + } catch { + session.activeConnIds.delete(connId) + await this.store.failReservation(pending.reservation) + if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId) + if (pending.credentialActivityId) { + this.releaseActivityBestEffort(identity, pending.credentialActivityId) + } + this.rejectClient(pending.client, RELAY_CLOSE_CODE.LIMIT_EXCEEDED) + socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'basis persistence failed') + return false + } + const close = wireSplice({ + client: pending.client, + host: socket, + budget: this.queuedByteBudget, + onClose: () => { + session.activeConnIds.delete(connId) + session.activeSplices.delete(connId) + this.deactivateBasisBestEffort(connId) + if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId) + if (pending.credentialActivityId) { + this.releaseActivityBestEffort(identity, pending.credentialActivityId) + } + }, + onForwardedBytes: (bytes) => this.observer.recordForwardedBytes(bytes), + onClosed: (closeInfo) => { + this.observer.recordSpliceClose?.(closeInfo.trigger) + // Only abnormal closes are logged; routine peer disconnects would be + // one line per phone backgrounding. + if ( + closeInfo.code === RELAY_CLOSE_CODE.LIMIT_EXCEEDED || + closeInfo.trigger.includes('error') || + closeInfo.trigger.includes('oversize') + ) { + console.warn( + `[orca-relay] splice closed host=${relayHostLogDigest(session.relayHostId)}` + + ` trigger=${closeInfo.trigger} code=${closeInfo.code}` + + ` reason=${JSON.stringify(closeInfo.reason)}` + ) + } + } + }) + session.activeSplices.set(connId, close) + if (pending.client.readyState !== pending.client.OPEN || socket.readyState !== socket.OPEN) { + close() + return false + } + send(pending.client, 'relay-hello', { + ok: true, + credentialKind: pending.reservation.credentialKind, + leaseExpiresAt: pending.reservation.leaseExpiresAt, + ...(pending.reservation.credentialKind === 'resume' + ? { + acceptedCredentialVersion: pending.reservation.acceptedCredentialVersion, + acceptedAs: pending.reservation.acceptedAs, + resumeExpiresAt: pending.reservation.resumeExpiresAt, + ...(pending.reservation.graceExpiresAt === undefined + ? {} + : { graceExpiresAt: pending.reservation.graceExpiresAt }) + } + : {}) + }) + return true + } + + acceptControl( + socket: WebSocket, + identity: RelayTokenClaims, + connectionInclusionWatermark?: number + ): void { + if (this.draining) { + socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining') + return + } + let firstFrameTimer: ReturnType | null = setTimeout(() => { + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host hello timeout') + }, 2_000) + socket.once('message', (raw, isBinary) => { + if (firstFrameTimer) clearTimeout(firstFrameTimer) + firstFrameTimer = null + if (isBinary) { + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host hello must be text') + return + } + this.guardSessionTask( + () => + this.beginProof( + socket, + identity, + payload(raw, 'host-hello'), + connectionInclusionWatermark + ), + socket, + 'host hello proof' + ) + }) + } + + // A dependency failure (e.g. a database connect timeout) must cost one + // handshake or command, not the process: an unhandled rejection here has + // crashed whole cells and wiped their in-memory draining flag. 4429 is the + // endpoint-scoped close in the contract, so the client retries this cell. + private guardSessionTask( + task: () => Promise, + socket: WebSocket | null, + context: string + ): void { + void Promise.resolve() + .then(task) + .catch((error: unknown) => { + const message = (error instanceof Error ? error.message : 'unknown') + // Untruncated, unlike peer-supplied close reasons: this is the + // primary diagnostic for the next rejection class. + .replace(/[^\x20-\x7e]/g, '') + console.warn(`[orca-relay] ${context} failed: ${message}`) + socket?.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'relay temporarily unavailable') + }) + // Terminal: a throw in the handler above must not itself crash the process. + .catch(() => {}) + } + + get(identity: RelayIdentityKey): HostSession | null { + return this.sessions.get(this.key(identity.userId, identity.relayHostId)) ?? null + } + + hasActiveControl(identity: RelayIdentityKey): boolean { + const session = this.get(identity) + return ( + session !== null && + session.state === 'active' && + session.socket !== null && + session.socket.readyState === session.socket.OPEN + ) + } + + runtimeCounts(): { controls: number; splices: number; pendingSplices: number } { + let controls = 0 + let splices = 0 + let pendingSplices = 0 + for (const session of this.sessions.values()) { + const socket = session.socket + if ( + socket !== null && + socket.readyState === socket.OPEN && + (session.state === 'active' || session.state === 'drain-only') + ) { + controls++ + } + splices += session.activeSplices.size + pendingSplices += session.pendingConns.size + } + return { controls, splices, pendingSplices } + } + + drain(graceMs: number): void { + this.draining = true + for (const session of this.sessions.values()) { + if (session.state === 'closed') continue + session.state = 'drain-only' + if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' }) + setTimeout(() => this.closeDrainedSession(session), graceMs) + } + } + + drainHost(input: { + attemptId: string + userId: string + relayHostId: string + sourceAssignmentEpoch: number + graceMs: number + }): RegionalHostDrainOutcome { + const session = this.get(input) + if (!session || session.state === 'closed') return 'host-not-connected' + if (session.assignmentEpoch !== input.sourceAssignmentEpoch) { + throw new Error('regional_rehome_assignment_epoch_mismatch') + } + if (session.regionalDrainAttemptId) { + if (session.regionalDrainAttemptId !== input.attemptId) { + throw new Error('regional_rehome_attempt_conflict') + } + this.reassertRegionalDrain(session) + return 'already-accepted' + } + session.regionalDrainAttemptId = input.attemptId + session.regionalDrainExpiresAt = this.now() + input.graceMs + this.reassertRegionalDrain(session) + session.regionalDrainTimer = setTimeout( + () => this.closeDrainedSession(session), + input.graceMs + ) + return 'accepted' + } + + isDraining(): boolean { + return this.draining + } + + private async beginProof( + socket: WebSocket, + identity: RelayTokenClaims, + candidate: unknown, + connectionInclusionWatermark?: number + ): Promise { + const hello = HostHelloSchema.safeParse(candidate) + const hostPublicKey = hello.success + ? decodeCanonicalBase64(hello.data.hostPublicKeyB64, 32) + : null + if (!hello.success) { + this.observer.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host hello') + return + } + if (!hostPublicKey) { + this.observer.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host public key') + return + } + if ( + hello.data.relayHostId !== identity.relayHostId || + relayHostId(hostPublicKey) !== identity.relayHostId + ) { + this.observer.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host key binding mismatch') + return + } + // Combined is staging-only compatibility; stamped cells require the durable director epoch. + const assignmentValid = + this.config.role === 'combined' + ? hello.data.assignmentEpoch === 1 + : await this.assignments.verifyCellAssignment({ + userId: identity.sub, + relayHostId: identity.relayHostId, + cellId: this.config.cellId, + assignmentEpoch: hello.data.assignmentEpoch + }) + if (!assignmentValid) { + this.observer.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch') + return + } + + const key = this.key(identity.sub, identity.relayHostId) + const existing = this.sessions.get(key) + const rebind = Boolean( + existing && + hello.data.controlResumeSecret && + hello.data.controlResumeSecret === existing.controlResumeSecret && + (existing.state === 'orphaned' || existing.state === 'active') + ) + const generation = rebind ? existing!.generation : (existing?.generation ?? 0) + 1 + const ephemeral = nacl.box.keyPair() + const challengeNonce = randomBytes(nacl.box.nonceLength) + const challengeSecret = randomBytes(32) + const challengeId = randomUUID() + const issuedAt = this.now() + const expiresAt = issuedAt + 10_000 + const transcript = buildHostProofTranscript({ + relayOrigin: this.config.publicUrl, + relayEphemeralPublicKey: ephemeral.publicKey, + challengeNonce, + challengeId, + issuedAt, + expiresAt, + userId: identity.sub, + profileId: identity.prof, + organizationId: identity.org ?? '', + relayHostId: identity.relayHostId, + hostPublicKey, + assignmentEpoch: hello.data.assignmentEpoch, + previousGeneration: hello.data.previousGeneration, + resumeRequested: rebind + }) + const plaintext = buildHostChallengePlaintext(transcript, challengeSecret) + const ciphertext = nacl.box(plaintext, challengeNonce, hostPublicKey, ephemeral.secretKey) + const expectedProof = createHmac('sha256', challengeSecret) + .update(buildHostProofMacInput(transcript)) + .digest() + send(socket, 'host-challenge', { + challengeId, + relayEphemeralPublicKeyB64: Buffer.from(ephemeral.publicKey).toString('base64'), + nonceB64: Buffer.from(challengeNonce).toString('base64'), + ciphertextB64: Buffer.from(ciphertext).toString('base64'), + expiresAt + }) + const proofTimer = setTimeout(() => { + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host proof timeout') + }, 10_000) + socket.once('message', (raw, isBinary) => { + clearTimeout(proofTimer) + const ack = isBinary ? null : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack')) + const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null + if ( + !ack?.success || + ack.data.challengeId !== challengeId || + !proof || + this.now() > expiresAt || + !timingSafeEqual(proof, expectedProof) + ) { + this.observer.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host proof') + return + } + this.observer.recordAuth(true) + this.guardSessionTask( + () => + this.activate( + socket, + identity, + existing ?? null, + generation, + rebind, + hello.data.assignmentEpoch, + hello.data.appVersion, + connectionInclusionWatermark + ), + socket, + 'host activation' + ) + }) + } + + private activate( + socket: WebSocket, + identity: RelayTokenClaims, + existing: HostSession | null, + generation: number, + rebind: boolean, + assignmentEpoch: number, + appVersion: string, + connectionInclusionWatermark?: number + ): Promise { + const key = this.key(identity.sub, identity.relayHostId) + const previous = this.activationQueues.get(key) ?? Promise.resolve() + // The timeout only fails this waiting socket; the queue entry still chains + // behind the stalled predecessor so activations never run concurrently. + let queueWaitExpired = false + const queueWaitTimer = setTimeout(() => { + queueWaitExpired = true + socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled') + }, ACTIVATION_QUEUE_WAIT_MS) + queueWaitTimer.unref?.() + const activation = previous.catch(() => undefined).then(async () => { + clearTimeout(queueWaitTimer) + if (queueWaitExpired) return + if ((this.sessions.get(key) ?? null) !== existing) { + socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded') + return + } + await this.activateCurrent( + socket, + identity, + existing, + generation, + rebind, + assignmentEpoch, + appVersion, + connectionInclusionWatermark + ) + }) + this.activationQueues.set(key, activation) + const cleanup = (): void => { + if (this.activationQueues.get(key) === activation) this.activationQueues.delete(key) + } + void activation.then(cleanup, cleanup) + return activation + } + + private async activateCurrent( + socket: WebSocket, + identity: RelayTokenClaims, + existing: HostSession | null, + generation: number, + rebind: boolean, + assignmentEpoch: number, + appVersion: string, + connectionInclusionWatermark?: number + ): Promise { + let controlActivityId: string | null = null + if (this.config.role === 'cell') { + try { + controlActivityId = await this.assignments.activateControl( + { userId: identity.sub, relayHostId: identity.relayHostId }, + { + cellId: this.config.cellId, + assignmentEpoch, + generation, + connectionInclusionWatermark + } + ) + await this.assignments.markMigrationTargetRegistered( + { userId: identity.sub, relayHostId: identity.relayHostId }, + { cellId: this.config.cellId, assignmentEpoch } + ) + } catch { + // A failure after activateControl succeeded must not strand the + // acquired control activity until lease expiry. + if (controlActivityId) { + this.releaseActivityBestEffort( + { userId: identity.sub, relayHostId: identity.relayHostId }, + controlActivityId + ) + } + socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'assignment changed during host proof') + return + } + } + if (this.draining || socket.readyState !== socket.OPEN) { + if (controlActivityId) { + this.releaseActivityBestEffort( + { userId: identity.sub, relayHostId: identity.relayHostId }, + controlActivityId + ) + } + if (socket.readyState === socket.OPEN) { + socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining') + } + return + } + if (existing) this.observer.recordReconnect() + if (rebind && existing) { + const previousSocket = existing.socket + if (existing.orphanTimer) clearTimeout(existing.orphanTimer) + existing.orphanTimer = null + existing.socket = socket + existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active' + existing.appVersion = appVersion + existing.leaseExpiresAt = this.now() + 55 * 60 * 1000 + existing.lastPongAt = this.now() + existing.activityRenewalDueAt = + this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + this.wireActiveControl(existing) + this.sendHelloAck(existing) + if (existing.regionalDrainAttemptId) this.reassertRegionalDrain(existing) + previousSocket?.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control rebound') + return + } + if (existing) { + existing.state = 'closed' + if (existing.heartbeatTimer) clearInterval(existing.heartbeatTimer) + if (existing.orphanTimer) clearTimeout(existing.orphanTimer) + if (existing.regionalDrainTimer) clearTimeout(existing.regionalDrainTimer) + existing.heartbeatTimer = null + existing.orphanTimer = null + existing.regionalDrainTimer = null + existing.closingCounts ??= { + splices: existing.activeSplices.size, + pending: existing.pendingConns.size + } + for (const close of existing.activeSplices.values()) close() + for (const pending of existing.pendingConns.values()) { + clearTimeout(pending.attachTimer) + pending.capacityReservation?.release() + this.failReservationBestEffort(pending.reservation) + if (pending.credentialActivityId) { + this.releaseActivityBestEffort( + { + userId: pending.reservation.userId, + relayHostId: pending.reservation.relayHostId + }, + pending.credentialActivityId + ) + } + this.rejectClient(pending.client, RELAY_CLOSE_CODE.PEER_DROPPED) + } + existing.socket?.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'replaced by a newer generation') + this.releaseControlActivity(existing) + } + const session: HostSession = { + identity, + relayHostId: identity.relayHostId, + generation, + assignmentEpoch, + controlActivityId, + controlResumeSecret: randomBytes(32).toString('base64url'), + appVersion, + state: 'active', + socket, + leaseExpiresAt: this.now() + 55 * 60 * 1000, + orphanTimer: null, + heartbeatTimer: null, + lastPongAt: this.now(), + activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs, + activityRenewalAttempt: 0, + activityRenewalCompletedAttempt: 0, + activeConnIds: new Set(), + activeSplices: new Map(), + pendingConns: new Map(), + closingCounts: null, + regionalDrainAttemptId: null, + regionalDrainTimer: null, + regionalDrainExpiresAt: null + } + this.sessions.set(this.key(identity.sub, identity.relayHostId), session) + this.wireActiveControl(session) + this.sendHelloAck(session) + } + + private wireActiveControl(session: HostSession): void { + const socket = session.socket! + const wiredAt = this.now() + // Why: pin the build to THIS socket. A rebind refreshes session.appVersion and + // only then closes the predecessor, whose close event always lands after that + // write, so reading it at log time would stamp the successor's build. + const appVersion = session.appVersion + let socketError: string | null = null + // Why: an unhandled ws 'error' (e.g. an oversize control frame) would + // otherwise throw process-wide; the message also explains the close below. + socket.on('error', (error) => { + socketError ??= error.message + }) + socket.once('close', (code, reason) => { + this.observer.recordControlClose?.(code) + // One line per control close makes reconnect churners attributable by + // host digest without exposing the raw relay host id. + console.warn( + `[orca-relay] control closed host=${relayHostLogDigest(session.relayHostId)}` + + ` gen=${session.generation} state=${session.state} ageMs=${this.now() - wiredAt}` + + ` app=${JSON.stringify(printableCloseReason(appVersion))}` + + ` splices=${session.closingCounts?.splices ?? session.activeSplices.size}` + + ` pending=${session.closingCounts?.pending ?? session.pendingConns.size}` + + ` code=${code} reason=${JSON.stringify(printableCloseReason(reason))}` + + (socketError === null ? '' : ` error=${JSON.stringify(printableCloseReason(socketError))}`) + ) + }) + socket.on('message', (raw, isBinary) => { + if (isBinary || (session.state !== 'active' && session.state !== 'drain-only')) return + try { + const parsed = JSON.parse(raw.toString()) as Record + if (parsed.type === 'pong') { + session.lastPongAt = this.now() + return + } + if (parsed.type === 'auth-refresh') { + // Close the socket the message arrived on: after a rebind, + // session.socket already points at the successor. + this.guardSessionTask(() => this.acceptRefresh(session, raw), socket, 'auth refresh') + return + } + this.guardSessionTask( + () => this.acceptControlCommand(session, parsed.type, raw), + socket, + 'control command' + ) + } catch { + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid control JSON') + } + }) + socket.once('close', () => { + if (session.socket !== socket || session.state === 'closed') return + session.socket = null + session.state = session.regionalDrainAttemptId ? 'drain-only' : 'orphaned' + if (session.heartbeatTimer) clearInterval(session.heartbeatTimer) + session.heartbeatTimer = null + session.orphanTimer = setTimeout(() => { + session.state = 'closed' + for (const close of session.activeSplices.values()) close() + const key = this.key(session.identity.sub, session.relayHostId) + if (this.sessions.get(key) === session) this.sessions.delete(key) + this.releaseControlActivity(session) + }, CONTROL_CONTINUITY_LIMITS.orphanGraceMs) + }) + if (session.heartbeatTimer) clearInterval(session.heartbeatTimer) + session.heartbeatTimer = setInterval( + () => this.heartbeat(session), + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + ) + } + + private async acceptRefresh(session: HostSession, raw: RawData): Promise { + const parsed = AuthRefreshSchema.safeParse(payload(raw, 'auth-refresh')) + if (!parsed.success) return + const refreshed = await this.verifyRelayToken(parsed.data.relayJwt) + const sameIdentity = + refreshed && + refreshed.sub === session.identity.sub && + refreshed.prof === session.identity.prof && + refreshed.org === session.identity.org && + refreshed.relayHostId === session.identity.relayHostId + if (!sameIdentity) { + session.socket?.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'refresh identity changed') + return + } + session.identity = refreshed + if (!session.regionalDrainAttemptId) session.state = 'active' + } + + private heartbeat(session: HostSession): void { + const key = this.key(session.identity.sub, session.relayHostId) + if (this.sessions.get(key) !== session || session.state === 'closed') { + if (session.heartbeatTimer) clearInterval(session.heartbeatTimer) + session.heartbeatTimer = null + return + } + const now = this.now() + if (!session.socket) return + const controlActivityId = session.controlActivityId + if (controlActivityId && now >= session.activityRenewalDueAt) { + const attempt = ++session.activityRenewalAttempt + const startedAt = now + void this.assignments + .renewControlActivity( + { userId: session.identity.sub, relayHostId: session.relayHostId }, + { + activityId: controlActivityId, + cellId: this.config.cellId, + expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS + } + ) + .then(() => { + if (attempt <= session.activityRenewalCompletedAttempt) return + session.activityRenewalCompletedAttempt = attempt + session.activityRenewalDueAt = startedAt + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS + }) + .catch(async (error: unknown) => { + if (error instanceof Error && error.message === 'activity_cell_not_authoritative') { + // Completion fences a late drain-only heartbeat after all source work is gone. + session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed') + return + } + if (error instanceof Error && error.message === 'control_activity_not_found') { + if ( + this.sessions.get(key) !== session || + session.state === 'closed' || + !session.socket + ) { + return + } + try { + await this.assignments.acquireActivity( + { userId: session.identity.sub, relayHostId: session.relayHostId }, + { + activityId: controlActivityId, + kind: 'control', + cellId: this.config.cellId + } + ) + this.observer.recordControlActivityRecovery?.(true) + } catch (acquireError: unknown) { + this.observer.recordControlActivityRecovery?.(false) + if ( + acquireError instanceof Error && + acquireError.message === 'activity_cell_not_authoritative' + ) { + session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed') + return + } + console.warn('[orca-relay] control activity recovery failed') + } + return + } + if (error instanceof Error && error.message === 'control_activity_moved') { + session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control activity moved') + return + } + console.warn('[orca-relay] control activity renewal failed') + }) + // Terminal handler: a throw inside the async catch above (e.g. a + // future await) must not become a process-killing rejection. + .catch(() => { + console.warn('[orca-relay] control activity renewal handling failed') + }) + } + if (now - session.lastPongAt > 75_000) { + session.socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control silence timeout') + return + } + const expiresAt = session.identity.exp * 1000 + if (now > expiresAt + CONTROL_CONTINUITY_LIMITS.expiredAuthExistingSpliceGraceMs) { + session.socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'relay authorization expired') + return + } + if (now > expiresAt) session.state = 'drain-only' + if (now > session.leaseExpiresAt) { + send(session.socket, 'drain', { graceMs: 0, recovery: 'resolve-director' }) + session.socket.close(RELAY_CLOSE_CODE.DRAINING, 'control lease expired') + return + } + send(session.socket, 'ping', { t: now }) + } + + private sendHelloAck(session: HostSession): void { + if (!session.socket) return + send(session.socket, 'host-hello-ack', { + v: 1, + generation: session.generation, + controlResumeSecret: session.controlResumeSecret, + leaseExpiresAt: session.leaseExpiresAt, + activeConnIds: [...session.activeConnIds], + pendingConns: [...session.pendingConns.values()].map((pending) => ({ + connId: pending.connId, + connTicket: pending.connTicket + })) + }) + } + + private closeDrainedSession(session: HostSession): void { + if (session.state === 'closed') return + if (session.heartbeatTimer) clearInterval(session.heartbeatTimer) + if (session.orphanTimer) clearTimeout(session.orphanTimer) + if (session.regionalDrainTimer) clearTimeout(session.regionalDrainTimer) + session.heartbeatTimer = null + session.orphanTimer = null + session.regionalDrainTimer = null + session.regionalDrainExpiresAt = null + session.closingCounts ??= { + splices: session.activeSplices.size, + pending: session.pendingConns.size + } + for (const close of session.activeSplices.values()) { + close(RELAY_CLOSE_CODE.DRAINING, 'relay draining') + } + for (const pending of session.pendingConns.values()) { + clearTimeout(pending.attachTimer) + pending.capacityReservation?.release() + this.failReservationBestEffort(pending.reservation) + if (pending.credentialActivityId) { + this.releaseActivityBestEffort( + { + userId: pending.reservation.userId, + relayHostId: pending.reservation.relayHostId + }, + pending.credentialActivityId + ) + } + this.rejectClient(pending.client, RELAY_CLOSE_CODE.DRAINING) + } + session.pendingConns.clear() + session.state = 'closed' + if (session.socket) { + closeRelayWebSocket( + session.socket, + RELAY_CLOSE_CODE.DRAINING, + 'resolve configured director' + ) + } + const key = this.key(session.identity.sub, session.relayHostId) + if (this.sessions.get(key) === session) this.sessions.delete(key) + this.releaseControlActivity(session) + } + + private reassertRegionalDrain(session: HostSession): void { + session.state = 'drain-only' + if (!session.socket) return + send(session.socket, 'drain', { + graceMs: Math.max(0, (session.regionalDrainExpiresAt ?? this.now()) - this.now()), + recovery: 'resolve-director' + }) + } + + private key(userId: string, hostId: string): string { + return `${userId}\0${hostId}` + } + + private async acceptControlCommand( + session: HostSession, + type: unknown, + raw: RawData + ): Promise { + if (typeof type !== 'string' || !session.socket) return + try { + const identity = { userId: session.identity.sub, relayHostId: session.relayHostId } + if (type === 'invite-create') { + if (session.state !== 'active') throw new Error('authorization_expired') + const request = InviteCreateSchema.parse(payload(raw, type)) + const activityId = `invite-offer:${request.reqId}` + if (this.config.role === 'cell') { + await this.assignments.acquireActivity(identity, { + activityId, + kind: 'invite', + cellId: this.config.cellId + }) + } + let invite + try { + invite = await this.store.createInvite(identity, request.relayDeviceId) + if (this.config.role === 'cell') { + await this.assignments.acquireActivity(identity, { + activityId, + kind: 'invite', + cellId: this.config.cellId, + expiresAt: invite.expiresAt + }) + } + } catch (error) { + if (this.config.role === 'cell') { + await this.assignments.releaseActivity(identity, activityId) + } + throw error + } + send(session.socket, 'invite-created', { reqId: request.reqId, ...invite }) + return + } + if (type === 'device-credential-install') { + const request = DeviceCredentialInstallSchema.parse(payload(raw, type)) + if ( + session.state !== 'active' && + request.authorization.mode === 'authenticated-direct' + ) { + throw new Error('authorization_expired') + } + const installActivityId = `install:${request.reqId}` + if (this.config.role === 'cell') { + await this.assignments.acquireActivity(identity, { + activityId: installActivityId, + kind: 'install', + cellId: this.config.cellId + }) + } + let result + try { + if (request.authorization.mode === 'authenticated-direct') { + await this.store.recordDirectAuthorization({ + ...identity, + relayDeviceId: request.relayDeviceId, + directAuthId: request.authorization.directAuthId, + owningControlGeneration: session.generation, + deadline: this.now() + RELAY_PROTOCOL_LIMITS.resumeConfirmationDeadlineMs + }) + } + result = await this.store.installCredential({ + ...identity, + ...request, + owningControlGeneration: session.generation + }) + } finally { + if (this.config.role === 'cell') { + await this.assignments.releaseActivity(identity, installActivityId) + } + } + if (request.authorization.mode === 'relay-basis') { + await this.assignments.releaseActivity( + identity, + `invite:${request.authorization.basisConnId}` + ) + } + send(session.socket, 'device-credential-installed', result) + return + } + if (type === 'device-credential-install-status') { + const request = DeviceCredentialInstallStatusSchema.parse(payload(raw, type)) + const result = await this.store.installStatus({ ...identity, ...request }) + send(session.socket, 'device-credential-install-status-result', { + v: 1, + reqId: request.reqId, + state: result ? 'committed' : 'not-found', + ...(result ? { result } : {}) + }) + return + } + if (type === 'device-resume-confirm') { + const request = DeviceResumeConfirmSchema.parse(payload(raw, type)) + const result = await this.store.confirmResume({ + ...identity, + ...request, + owningControlGeneration: session.generation + }) + await this.assignments.releaseActivity(identity, `confirmation:${request.basisConnId}`) + send(session.socket, 'device-resume-confirmed', result) + return + } + if (type === 'device-revoke') { + const request = DeviceRevokeSchema.parse(payload(raw, type)) + await this.store.revoke(identity, request.relayDeviceId) + send(session.socket, 'device-revoked', { reqId: request.reqId }) + return + } + this.sendControlError(session, undefined, 'unknown_control_message') + } catch (error) { + const reqId = (() => { + const candidate = payload(raw, type) + return typeof candidate === 'object' && candidate && 'reqId' in candidate + ? String(candidate.reqId) + : undefined + })() + this.sendControlError( + session, + reqId, + error instanceof Error ? error.message : 'control_operation_failed' + ) + } + } + + private sendControlError(session: HostSession, reqId: string | undefined, code: string): void { + if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code }) + } + + private rejectClient(socket: WebSocket, code: number): void { + send(socket, 'relay-hello', { ok: false, code }) + closeRelayWebSocket(socket, code, 'relay connection rejected') + } + + private releaseControlActivity(session: HostSession): void { + if (!session.controlActivityId) return + this.releaseActivityBestEffort( + { userId: session.identity.sub, relayHostId: session.relayHostId }, + session.controlActivityId + ) + } + + private releaseActivityBestEffort(identity: RelayIdentityKey, activityId: string): void { + void this.assignments.releaseActivity(identity, activityId).catch(() => { + // Why: expiry cleanup is the durable fallback; a transient SQL failure while + // closing a socket must not become an unhandled rejection that kills the cell. + console.warn('[orca-relay] activity release deferred to lease cleanup') + }) + } + + private failReservationBestEffort(reservation: CredentialReservation): void { + // Why: reservation deadlines and basis cleanup are durable recovery paths; + // transient SQL errors during socket callbacks must stay process-contained. + void this.store.failReservation(reservation).catch(() => { + console.warn('[orca-relay] reservation release deferred to credential cleanup') + }) + } + + private deactivateBasisBestEffort(connId: string): void { + void this.store.deactivateBasis(connId).catch(() => { + console.warn('[orca-relay] basis deactivation deferred to credential cleanup') + }) + } +} + +export type RelayIdentityKey = { userId: string; relayHostId: string } diff --git a/cloud/apps/relay/src/index.ts b/cloud/apps/relay/src/index.ts new file mode 100644 index 00000000000..635f3ae9b38 --- /dev/null +++ b/cloud/apps/relay/src/index.ts @@ -0,0 +1,131 @@ +import { + formatAssignmentInventorySnapshot, + readAssignmentInventorySnapshot +} from './assignment-inventory-snapshot.js' +import { RelayAssignmentStore } from './assignment-store.js' +import { loadRelayConfig } from './config.js' +import { startCellHeartbeat } from './cell-heartbeat-client.js' +import { + reconcileCellAdmissionAtStartup, + roleOwnsAssignmentMaintenance +} from './cell-admission-startup.js' +import { + consumeRelayDatabasePoolPressure, + openRelayDatabase, + readRelayDatabasePoolPressure +} from './database.js' +import { runAssignmentCleanup } from './assignment-cleanup-steps.js' +import { runRelayBackgroundOperation } from './relay-background-operation.js' +import { observedRelayRequests } from './relay-observability.js' +import { startRegionalRehomeWorker } from './regional-rehome-worker.js' +import { createRelayServer } from './relay-server.js' +import { + formatRegisteredMigrationInventory, + readRegisteredMigrationInventory +} from './registered-migration-inventory.js' + +const config = loadRelayConfig() +const database = await openRelayDatabase({ + databaseUrl: config.databaseUrl, + dataDir: config.dataDir, + poolMax: config.databasePoolMax, + applicationName: `orca-relay/${config.role}/${config.cellId}` +}) +await reconcileCellAdmissionAtStartup(config, new RelayAssignmentStore(database)) +const { + server, + sessions, + store, + assignments, + observability, + runtimeCounts, + connectionSnapshot, + ready, + cellIncarnation +} = createRelayServer(config, database) +const cleanupTimer = setInterval( + () => + void runRelayBackgroundOperation( + () => store.cleanup(), + '[orca-relay] credential cleanup failed' + ), + 30_000 +) +const assignmentCleanupTimer = roleOwnsAssignmentMaintenance(config.role) + ? setInterval(() => { + void runAssignmentCleanup(assignments) + }, 30_000) + : null +const inventorySnapshotTimer = roleOwnsAssignmentMaintenance(config.role) + ? setInterval(() => { + void runRelayBackgroundOperation(async () => { + const snapshot = await readAssignmentInventorySnapshot(database, Date.now()) + for (const line of formatAssignmentInventorySnapshot(snapshot)) console.warn(line) + }, '[orca-relay] inventory snapshot failed') + }, 60_000) + : null +const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role) + ? setInterval(() => { + void runRelayBackgroundOperation(async () => { + const inventory = await readRegisteredMigrationInventory(database, Date.now()) + for (const line of formatRegisteredMigrationInventory(inventory)) console.warn(line) + }, '[orca-relay] migration inventory failed') + }, 5 * 60_000) + : null +cleanupTimer.unref() +assignmentCleanupTimer?.unref() +inventorySnapshotTimer?.unref() +migrationInventoryTimer?.unref() +observability.start(() => ({ + ...runtimeCounts(), + ...consumeRelayDatabasePoolPressure(database) +})) +const regionalRehomeWorker = startRegionalRehomeWorker(config, assignments, { + safetySnapshot: () => ({ + ...observability.regionalRehomeRuntimeSafety(), + ...readRelayDatabasePoolPressure(database) + }) +}) +const heartbeat = startCellHeartbeat(config, { + ready, + incarnation: cellIncarnation, + observedRequests: () => observedRelayRequests(runtimeCounts()), + connectionCounts: () => { + const snapshot = connectionSnapshot() + const counts = runtimeCounts() + return { + totalConnections: snapshot?.physicalConnections ?? counts.totalConnections, + inFlightConnections: + snapshot?.inFlightConnections ?? counts.inFlightConnections ?? 0, + reservedConnectionUnits: + snapshot?.reservedConnectionUnits ?? counts.reservedConnectionUnits ?? 0, + enforcedConnectionUnits: + snapshot?.enforcedConnectionUnits ?? + counts.enforcedConnectionUnits ?? + counts.totalConnections, + inclusionWatermark: snapshot?.inclusionWatermark + } + }, + regionalRehomeSafety: () => ({ + ...observability.regionalRehomeRuntimeSafety(), + ...readRelayDatabasePoolPressure(database) + }) +}) + +server.listen(config.port, () => { + console.log(`[orca-relay] listening on ${config.publicUrl} (port ${config.port})`) +}) + +const shutdown = (): void => { + clearInterval(cleanupTimer) + if (assignmentCleanupTimer) clearInterval(assignmentCleanupTimer) + if (inventorySnapshotTimer) clearInterval(inventorySnapshotTimer) + if (migrationInventoryTimer) clearInterval(migrationInventoryTimer) + observability.stop() + heartbeat?.stop() + regionalRehomeWorker?.stop() + sessions.drain(0) + server.close(() => void database.close()) +} +process.once('SIGTERM', shutdown) +process.once('SIGINT', shutdown) diff --git a/cloud/apps/relay/src/migration-recovery-postgres.test.ts b/cloud/apps/relay/src/migration-recovery-postgres.test.ts new file mode 100644 index 00000000000..aee41278d49 --- /dev/null +++ b/cloud/apps/relay/src/migration-recovery-postgres.test.ts @@ -0,0 +1,1376 @@ +import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + RelayAssignmentStore, + STRANDED_MIGRATION_ABANDON_MS, + type RelayAssignmentMigration +} from './assignment-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' +import { readRegisteredMigrationInventory } from './registered-migration-inventory.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip + +type RecoveryFixture = { + store: RelayAssignmentStore + identity: { userId: string; relayHostId: string } + migration: RelayAssignmentMigration + sourceControlId: string + targetControlId: string + cellIncarnation: string + cells: { + source: { id: string; url: string; capacityRequests: number } + failed: { id: string; url: string; capacityRequests: number } + replacement: { id: string; url: string; capacityRequests: number } + } + advancePastHeartbeat: () => void + advance: (milliseconds: number) => void + heartbeat: (cell: { id: string; url: string }) => Promise +} + +describePostgres('PostgreSQL migration recovery', () => { + let database: RelayDatabase + let sequence = 0 + + beforeAll(async () => { + database = await openRelayDatabase({ databaseUrl, dataDir: '' }) + }) + + afterAll(async () => { + await database.query( + `DELETE FROM relay_control_connection_reservations + WHERE user_id LIKE 'recovery-user-%'` + ) + await database.query( + `DELETE FROM relay_post_drain_migration_pins WHERE user_id LIKE 'recovery-user-%'` + ) + await database.query( + `DELETE FROM relay_assignment_migration_incarnations WHERE user_id LIKE 'recovery-user-%'` + ) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'recovery-user-%'` + ) + await database.query( + `DELETE FROM relay_assignment_migrations WHERE user_id LIKE 'recovery-user-%'` + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'recovery-user-%'`) + await database.query( + `DELETE FROM relay_cell_fence_apply_invocations WHERE attempt_id IN ( + SELECT attempt_id FROM relay_cell_fence_attempts + WHERE cell_id LIKE 'recovery-cell-%' + )` + ) + await database.query( + `DELETE FROM relay_cell_committed_fences WHERE cell_id LIKE 'recovery-cell-%'` + ) + await database.query( + `DELETE FROM relay_cell_legacy_fence_adoptions + WHERE cell_id LIKE 'recovery-cell-%'` + ) + await database.query( + `DELETE FROM relay_cell_fence_plan_bindings WHERE attempt_id IN ( + SELECT attempt_id FROM relay_cell_fence_attempts + WHERE cell_id LIKE 'recovery-cell-%' + )` + ) + await database.query( + `DELETE FROM relay_cell_fence_attempts WHERE cell_id LIKE 'recovery-cell-%'` + ) + await database.query(`DELETE FROM relay_cell_fences WHERE cell_id LIKE 'recovery-cell-%'`) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id LIKE 'recovery-cell-%'`) + await database.query(`DELETE FROM relay_cell_admission WHERE cell_id LIKE 'recovery-cell-%'`) + await database.query(`DELETE FROM relay_cells WHERE cell_id LIKE 'recovery-cell-%'`) + await database.close() + }) + + async function fixture(replacementCapacity = 20): Promise { + sequence++ + let now = 100 + const suffix = String(sequence) + const cellIncarnation = `11111111-1111-4111-8111-${suffix.padStart(12, '0')}` + const cells = { + source: { + id: `recovery-cell-${suffix}-source`, + url: `https://recovery-${suffix}-source.example.com`, + capacityRequests: 20 + }, + failed: { + id: `recovery-cell-${suffix}-failed`, + url: `https://recovery-${suffix}-failed.example.com`, + capacityRequests: 20 + }, + replacement: { + id: `recovery-cell-${suffix}-replacement`, + url: `https://recovery-${suffix}-replacement.example.com`, + capacityRequests: replacementCapacity + } + } + const store = new RelayAssignmentStore(database, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells(Object.values(cells), false) + const heartbeat = async (cell: { id: string; url: string }): Promise => { + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation, + startedAt: 50, + ready: true, + observedRequests: 0 + }) + } + for (const cell of Object.values(cells)) await heartbeat(cell) + const identity = { + userId: `recovery-user-${suffix}`, + relayHostId: `recoveryhost${suffix.padStart(4, '0')}` + } + const sourceControlId = `control:${cells.source.id}:1` + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + cells.source.id, + 1, + 90_100, + now, + 1, + 0, + 0, + 0, + 0, + 0 + ] + ) + await database.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + identity.userId, + identity.relayHostId, + sourceControlId, + 'control', + cells.source.id, + 1, + 90_100, + now + ] + ) + await database.query( + `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = ?`, + [cells.source.id] + ) + await store.setCellEnabled(cells.source.id, false) + const migration = await store.startEvacuation(identity, cells.failed.id) + const targetControlId = await store.activateControl(identity, { + cellId: cells.failed.id, + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: cells.failed.id, + assignmentEpoch: migration.assignmentEpoch + }) + return { + store, + identity, + migration, + sourceControlId, + targetControlId, + cellIncarnation, + cells, + advancePastHeartbeat: () => { + now += 45_001 + }, + advance: (milliseconds) => { + now += milliseconds + }, + heartbeat + } + } + + async function completeSourceFence(input: RecoveryFixture): Promise { + const attemptId = input.cellIncarnation + const invocationId = input.cellIncarnation + const requestReason = `relay-recovery-test/${attemptId}` + const evidence = { + attemptId, + environment: 'production' as const, + cellId: input.cells.source.id, + cellIncarnation: input.cellIncarnation, + migName: input.cells.source.id, + instanceGroup: `https://compute.example/instanceGroups/${input.cells.source.id}`, + generationIdentity: `https://compute.example/instanceTemplates/${input.cells.source.id}`, + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + planObjectName: `relay-fence-plans/${attemptId}.tfplan`, + planObjectGeneration: '1', + varFileSha256: 'c'.repeat(64), + terraformStateLineage: input.cellIncarnation, + terraformStateSerial: 1, + terraformStateObjectGeneration: '1', + terraformStateObjectSha256: 'd'.repeat(64), + requestReason + } + await input.store.prepareCellFenceAttempt(evidence) + await input.store.bindCellFencePlanGeneration(evidence, evidence.planObjectGeneration) + await input.store.startCellFenceApply( + evidence, + invocationId, + `${requestReason}/${invocationId}` + ) + await input.store.recordCellFenceOperation( + evidence, + invocationId, + `${requestReason}/${invocationId}`, + `operation-${input.cells.source.id}` + ) + await input.store.attestCellFenceAttempt( + evidence, + `operation-${input.cells.source.id}` + ) + } + + it('reaps an abandoned registered migration onto its healthy target', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advance( + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + ) + + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + expect( + await database.query( + `SELECT assignment.cell_id, assignment.assignment_epoch, + migration.completed_at, migration.aborted_at + FROM relay_assignments assignment + JOIN relay_assignment_migrations migration + ON migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + AND migration.assignment_epoch = assignment.assignment_epoch + WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + ).toEqual([ + { + cell_id: input.cells.failed.id, + assignment_epoch: '2', + completed_at: String( + 100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + ), + aborted_at: null + } + ]) + }) + + it('reaps immediately after the retired source has a durable completed fence', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await completeSourceFence(input) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + expect( + await database.query( + `SELECT assignment.cell_id, assignment.assignment_epoch, + migration.completed_at, migration.aborted_at + FROM relay_assignments assignment + JOIN relay_assignment_migrations migration + ON migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + AND migration.assignment_epoch = assignment.assignment_epoch + WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + ).toEqual([ + { + cell_id: input.cells.failed.id, + assignment_epoch: '2', + completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1), + aborted_at: null + } + ]) + }) + + it('reaps immediately after the retired source has an adopted legacy fence', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await input.store.adoptLegacyCellFence( + input.cells.source.id, + input.cellIncarnation + ) + await input.store.commitLegacyCellFenceAdoption( + input.cells.source.id, + input.cellIncarnation + ) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + + expect( + ( + await readRegisteredMigrationInventory( + database, + 100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + ) + ).abandoned + ).toBe(1) + + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + expect( + await database.query( + `SELECT assignment.cell_id, assignment.assignment_epoch, + migration.completed_at, migration.aborted_at + FROM relay_assignments assignment + JOIN relay_assignment_migrations migration + ON migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + AND migration.assignment_epoch = assignment.assignment_epoch + WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + ).toEqual([ + { + cell_id: input.cells.failed.id, + assignment_epoch: '2', + completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1), + aborted_at: null + } + ]) + }) + + it('does not reap after temporary legacy adoption without durable commit', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await input.store.adoptLegacyCellFence( + input.cells.source.id, + input.cellIncarnation + ) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + + expect( + ( + await readRegisteredMigrationInventory( + database, + 100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1 + ) + ).abandoned + ).toBe(0) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0) + input.advance(STRANDED_MIGRATION_ABANDON_MS) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + }) + + it('invalidates an adopted legacy fence when the source heartbeats', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await input.store.adoptLegacyCellFence( + input.cells.source.id, + input.cellIncarnation + ) + await input.store.commitLegacyCellFenceAdoption( + input.cells.source.id, + input.cellIncarnation + ) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + await input.heartbeat(input.cells.source) + + expect( + await database.query( + `SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + [input.cells.source.id] + ) + ).toEqual([]) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0) + input.advance(STRANDED_MIGRATION_ABANDON_MS) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + }) + + it('invalidates an adopted legacy fence when the source restarts', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await input.store.adoptLegacyCellFence( + input.cells.source.id, + input.cellIncarnation + ) + await input.store.commitLegacyCellFenceAdoption( + input.cells.source.id, + input.cellIncarnation + ) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + await input.store.recordCellHeartbeat({ + cellId: input.cells.source.id, + cellUrl: input.cells.source.url, + cellIncarnation: '99999999-9999-4999-8999-999999999999', + startedAt: 51, + ready: true, + observedRequests: 0 + }) + + expect( + await database.query( + `SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + [input.cells.source.id] + ) + ).toEqual([]) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0) + input.advance(STRANDED_MIGRATION_ABANDON_MS) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + }) + + it('serializes legacy adoption with a returning source heartbeat', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await input.store.adoptLegacyCellFence( + input.cells.source.id, + input.cellIncarnation + ) + + const [commit, heartbeatResult] = await Promise.allSettled([ + input.store.commitLegacyCellFenceAdoption( + input.cells.source.id, + input.cellIncarnation + ), + input.heartbeat(input.cells.source) + ]) + expect(heartbeatResult.status).toBe('fulfilled') + expect(['fulfilled', 'rejected']).toContain(commit.status) + expect( + await database.query( + `SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`, + [input.cells.source.id] + ) + ).toEqual([]) + input.advance( + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + ) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + }) + + it('restores the 24-hour guard when the fenced source heartbeats again', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await completeSourceFence(input) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + await input.heartbeat(input.cells.source) + + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0) + expect( + await database.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + ).toEqual([{ completed_at: null, aborted_at: null }]) + input.advance(STRANDED_MIGRATION_ABANDON_MS) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + }) + + it('restores the 24-hour guard when the fenced source restarts', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await completeSourceFence(input) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + await input.store.recordCellHeartbeat({ + cellId: input.cells.source.id, + cellUrl: input.cells.source.url, + cellIncarnation: '99999999-9999-4999-8999-999999999999', + startedAt: 51, + ready: true, + observedRequests: 0 + }) + + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0) + expect( + await database.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + ).toEqual([{ completed_at: null, aborted_at: null }]) + input.advance(STRANDED_MIGRATION_ABANDON_MS) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + }) + + it('serializes fenced cleanup with a returning source heartbeat', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advancePastHeartbeat() + await completeSourceFence(input) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + + const [cleanup, heartbeatResult] = await Promise.allSettled([ + input.store.abortExpiredEvacuations(), + input.heartbeat(input.cells.source) + ]) + expect(cleanup.status).toBe('fulfilled') + expect(heartbeatResult.status).toBe('fulfilled') + if (cleanup.status !== 'fulfilled') throw cleanup.reason + if (heartbeatResult.status !== 'fulfilled') throw heartbeatResult.reason + expect([0, 1]).toContain(cleanup.value) + const migrations = await database.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + if (cleanup.value === 1) { + expect(migrations[0]?.completed_at).not.toBeNull() + } else { + expect(migrations).toEqual([{ completed_at: null, aborted_at: null }]) + input.advance(STRANDED_MIGRATION_ABANDON_MS) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + } + }) + + it('keeps a freshly retired source protected without a durable fence', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1) + + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0) + expect( + await database.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + ).toEqual([{ completed_at: null, aborted_at: null }]) + input.advance(STRANDED_MIGRATION_ABANDON_MS) + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + }) + + it('rolls an abandoned disabled target back to its source', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advance( + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + ) + + await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1) + expect( + await database.query( + `SELECT assignment.cell_id, assignment.assignment_epoch, + migration.completed_at, migration.aborted_at + FROM relay_assignments assignment + JOIN relay_assignment_migrations migration + ON migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + WHERE assignment.user_id = ? AND assignment.relay_host_id = ? + ORDER BY migration.assignment_epoch`, + [input.identity.userId, input.identity.relayHostId] + ) + ).toEqual([ + { + cell_id: input.cells.source.id, + assignment_epoch: '3', + completed_at: null, + aborted_at: String( + 100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + ) + } + ]) + }) + + it('serializes cleanup against supersession without reporting a false target', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + await input.store.releaseActivity(input.identity, input.targetControlId) + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advance( + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1 + ) + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation) + + const [cleanup, supersession] = await Promise.allSettled([ + input.store.abortExpiredEvacuations(), + input.store.supersedeRegisteredCellEvacuations( + input.cells.source.id, + input.cells.failed.id, + input.cells.replacement.id, + 100 + ) + ]) + expect(cleanup.status).toBe('fulfilled') + if (cleanup.status !== 'fulfilled') throw cleanup.reason + if (supersession.status === 'fulfilled') { + expect([0, 1]).toContain(supersession.value) + const assignment = await database.query( + `SELECT cell_id FROM relay_assignments WHERE user_id = ?`, + [input.identity.userId] + ) + expect(assignment).toEqual([{ + cell_id: + supersession.value === 1 + ? input.cells.replacement.id + : input.cells.source.id + }]) + if (supersession.value === 0) expect(cleanup.value).toBeGreaterThanOrEqual(1) + } else { + expect(cleanup.value).toBeGreaterThanOrEqual(1) + expect(supersession.reason).toMatchObject({ message: 'migration_already_superseded' }) + expect( + await database.query( + `SELECT cell_id FROM relay_assignments WHERE user_id = ?`, + [input.identity.userId] + ) + ).toEqual([{ cell_id: input.cells.source.id }]) + } + }) + + it('completes a registered migration from a dead source idempotently', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.failed) + await input.store.attestCellFence( + input.cells.source.id, + input.cellIncarnation + ) + const completion = { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + targetCellId: input.cells.failed.id + } + + await expect( + input.store.cellEvacuationStatus( + input.cells.source.id, + input.cells.failed.id, + true + ) + ).resolves.toMatchObject({ inProgress: 0, completed: 1 }) + await expect( + input.store.completeEvacuationFromDeadSource(input.identity, completion) + ).resolves.toMatchObject({ changed: false }) + expect(await reservations(input.cells)).toEqual({ source: 0, failed: 1, replacement: 0 }) + }) + + it('fails dead-source completion without a stale source heartbeat', async () => { + const input = await fixture() + await input.store.releaseActivity(input.identity, input.sourceControlId) + + await expect( + input.store.completeEvacuationFromDeadSource(input.identity, { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + targetCellId: input.cells.failed.id + }) + ).rejects.toThrow('cell_fence_attestation_missing') + expect(await reservations(input.cells)).toEqual({ source: 0, failed: 2, replacement: 0 }) + }) + + it('supersedes a registered migration with exact accounting and idempotency', async () => { + const input = await fixture() + await database.query( + `INSERT INTO relay_control_connection_reservations + (reservation_id, idempotency_key, user_id, relay_host_id, + assignment_epoch, cell_id, state, inclusion_watermark, + claim_activity_id, created_at, timeout_at, claimed_at, released_at, + updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'late-arrival-debt', NULL, NULL, 100, 100, NULL, NULL, 100)`, + [ + `superseded-${input.identity.userId}`, + `superseded-${input.identity.userId}`, + input.identity.userId, + input.identity.relayHostId, + input.migration.assignmentEpoch, + input.cells.failed.id + ] + ) + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence( + input.cells.failed.id, + input.cellIncarnation + ) + const supersession = { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + currentTargetCellId: input.cells.failed.id, + replacementTargetCellId: input.cells.replacement.id + } + + const first = await input.store.supersedeRegisteredEvacuation( + input.identity, + supersession + ) + const retry = await input.store.supersedeRegisteredEvacuation( + input.identity, + supersession + ) + expect(retry).toEqual(first) + expect(first).toMatchObject({ previousEpoch: 2, assignmentEpoch: 3 }) + expect( + await database.query( + `SELECT state FROM relay_control_connection_reservations + WHERE reservation_id = ?`, + [`superseded-${input.identity.userId}`] + ) + ).toEqual([{ state: 'released' }]) + expect(await reservations(input.cells)).toEqual({ source: 1, failed: 0, replacement: 2 }) + expect( + await database.query( + `SELECT assignment_epoch, completed_at, aborted_at + FROM relay_assignment_migrations WHERE user_id = ? + ORDER BY assignment_epoch`, + [input.identity.userId] + ) + ).toEqual([ + { assignment_epoch: '2', completed_at: null, aborted_at: '45101' }, + { assignment_epoch: '3', completed_at: null, aborted_at: null } + ]) + }) + + it('reconciles durable cell accounting before aggregate supersession', async () => { + const input = await fixture() + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence( + input.cells.failed.id, + input.cellIncarnation + ) + await database.query( + `UPDATE relay_cells + SET reserved_requests = CASE WHEN cell_id = ? THEN 1 ELSE 0 END + WHERE cell_id IN (?, ?)`, + [input.cells.replacement.id, input.cells.source.id, input.cells.replacement.id] + ) + + await expect( + input.store.supersedeRegisteredCellEvacuations( + input.cells.source.id, + input.cells.failed.id, + input.cells.replacement.id, + 100 + ) + ).resolves.toBe(1) + expect(await reservations(input.cells)).toEqual({ + source: 1, + failed: 0, + replacement: 2 + }) + }) + + it('rebuilds an expired registered migration lease before supersession', async () => { + const input = await fixture() + await pinMigration(input, input.migration.assignmentEpoch) + await clearActivities(input) + await database.query( + `UPDATE relay_assignment_migrations SET expires_at = 0 + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch] + ) + await fenceFailedCell(input) + + await expect(supersedeAggregate(input)).resolves.toBe(1) + expect(await reservations(input.cells)).toEqual({ + source: 0, + failed: 0, + replacement: 1 + }) + expect( + await database.query( + `SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at + FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`, + [input.identity.userId] + ) + ).toEqual([ + { + assignment_epoch: '2', + source_request_units: '1', + target_reserved_units: '2', + aborted_at: '45101' + }, + { + assignment_epoch: '3', + source_request_units: '0', + target_reserved_units: '1', + aborted_at: null + } + ]) + }) + + it('retires an obsolete row without discarding replacement activity', async () => { + const input = await fixture() + await pinMigration(input, input.migration.assignmentEpoch) + await fenceFailedCell(input) + await input.store.supersedeRegisteredEvacuation(input.identity, { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + currentTargetCellId: input.cells.failed.id, + replacementTargetCellId: input.cells.replacement.id + }) + await database.query( + `UPDATE relay_assignment_migrations SET aborted_at = NULL + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch] + ) + const activityBefore = await assignmentActivities(input) + + await expect(supersedeAggregate(input)).resolves.toBe(1) + expect(await assignmentActivities(input)).toEqual(activityBefore) + expect(await reservations(input.cells)).toEqual({ + source: 1, + failed: 0, + replacement: 2 + }) + expect( + await database.query( + `SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? ORDER BY assignment_epoch`, + [input.identity.userId] + ) + ).toEqual([ + { assignment_epoch: '2', aborted_at: '45101' }, + { assignment_epoch: '3', aborted_at: null } + ]) + }) + + it('anchors a newer dormant failed-cell epoch and preserves drain evidence', async () => { + const input = await fixture() + const drainAttemptId = await pinMigration(input, input.migration.assignmentEpoch) + await clearActivities(input) + await database.query( + `UPDATE relay_assignments SET assignment_epoch = 4 + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + await database.query( + `UPDATE relay_assignment_migrations SET expires_at = 0 + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch] + ) + await fenceFailedCell(input) + + await expect(supersedeAggregate(input)).resolves.toBe(1) + expect(await reservations(input.cells)).toEqual({ + source: 0, + failed: 0, + replacement: 1 + }) + expect( + await database.query( + `SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at + FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`, + [input.identity.userId] + ) + ).toEqual([ + { + assignment_epoch: '2', + source_request_units: '1', + target_reserved_units: '2', + aborted_at: '45101' + }, + { + assignment_epoch: '4', + source_request_units: '0', + target_reserved_units: '1', + aborted_at: '45101' + }, + { + assignment_epoch: '5', + source_request_units: '0', + target_reserved_units: '1', + aborted_at: null + } + ]) + expect( + await database.query( + `SELECT assignment_epoch, drain_attempt_id, source_request_units, + target_reserved_units + FROM relay_post_drain_migration_pins + WHERE user_id = ? ORDER BY assignment_epoch`, + [input.identity.userId] + ) + ).toEqual([ + { + assignment_epoch: '2', + drain_attempt_id: drainAttemptId, + source_request_units: '1', + target_reserved_units: '2' + }, + { + assignment_epoch: '4', + drain_attempt_id: drainAttemptId, + source_request_units: '0', + target_reserved_units: '1' + }, + { + assignment_epoch: '5', + drain_attempt_id: drainAttemptId, + source_request_units: '0', + target_reserved_units: '1' + } + ]) + }) + + it('uses an existing current-epoch migration once when retiring an older row', async () => { + const input = await fixture() + await pinMigration(input, input.migration.assignmentEpoch) + await clearActivities(input) + await database.query( + `UPDATE relay_assignments SET assignment_epoch = 4 + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + await database.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, + previous_epoch, assignment_epoch, source_request_units, + target_reserved_units, expires_at, target_registered_at, + completed_at, aborted_at, created_at, updated_at) + VALUES (?, ?, ?, ?, 2, 4, 0, 1, 0, 100, NULL, NULL, 100, 100)`, + [ + input.identity.userId, + input.identity.relayHostId, + input.cells.source.id, + input.cells.failed.id + ] + ) + await database.query( + `INSERT INTO relay_assignment_migration_incarnations + (user_id, relay_host_id, assignment_epoch, source_cell_incarnation, + target_cell_incarnation) + VALUES (?, ?, 4, ?, ?)`, + [ + input.identity.userId, + input.identity.relayHostId, + input.cellIncarnation, + input.cellIncarnation + ] + ) + await pinMigration(input, 4) + await fenceFailedCell(input) + + await expect(supersedeAggregate(input)).resolves.toBe(1) + expect( + await database.query( + `SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? ORDER BY assignment_epoch`, + [input.identity.userId] + ) + ).toEqual([ + { assignment_epoch: '2', aborted_at: '45101' }, + { assignment_epoch: '4', aborted_at: '45101' }, + { assignment_epoch: '5', aborted_at: null } + ]) + expect(await reservations(input.cells)).toEqual({ + source: 0, + failed: 0, + replacement: 1 + }) + }) + + it('rejects a newer failed-cell epoch with ambiguous activity', async () => { + const input = await fixture() + await database.query( + `UPDATE relay_assignments SET assignment_epoch = 4 + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + await fenceFailedCell(input) + + await expect(supersedeAggregate(input)).rejects.toThrow( + 'migration_activity_topology_mismatch' + ) + expect( + await database.query( + `SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations + WHERE user_id = ?`, + [input.identity.userId] + ) + ).toEqual([{ assignment_epoch: '2', aborted_at: null }]) + expect(await reservations(input.cells)).toEqual({ + source: 1, + failed: 2, + replacement: 0 + }) + }) + + it('leaves lease repair retryable when supersession later fails', async () => { + const input = await fixture(1) + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'migration'`, + [input.identity.userId, input.identity.relayHostId] + ) + await database.query( + `UPDATE relay_assignments SET migration_leases = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + await database.query( + `UPDATE relay_cells SET reserved_requests = reserved_requests - 1 + WHERE cell_id = ?`, + [input.cells.failed.id] + ) + await database.query( + `UPDATE relay_assignment_migrations SET expires_at = 0 + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch] + ) + await fenceFailedCell(input) + + await expect(supersedeAggregate(input)).rejects.toThrow( + 'relay_capacity_exhausted' + ) + expect( + await database.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND activity_kind = 'migration'`, + [input.identity.userId] + ) + ).toEqual([{ activity_id: 'migration:2' }]) + expect( + await database.query( + `SELECT aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND assignment_epoch = 2`, + [input.identity.userId] + ) + ).toEqual([{ aborted_at: null }]) + + await database.query( + `UPDATE relay_cells SET capacity_requests = 2 WHERE cell_id = ?`, + [input.cells.replacement.id] + ) + await expect(supersedeAggregate(input)).resolves.toBe(1) + expect(await reservations(input.cells)).toEqual({ + source: 1, + failed: 0, + replacement: 2 + }) + }) + + it('serializes concurrent supersession retries without duplicating capacity', async () => { + const input = await fixture() + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence( + input.cells.failed.id, + input.cellIncarnation + ) + const supersession = { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + currentTargetCellId: input.cells.failed.id, + replacementTargetCellId: input.cells.replacement.id + } + + const results = await Promise.all([ + input.store.supersedeRegisteredEvacuation(input.identity, supersession), + input.store.supersedeRegisteredEvacuation(input.identity, supersession) + ]) + expect(results[0]).toEqual(results[1]) + expect(await reservations(input.cells)).toEqual({ source: 1, failed: 0, replacement: 2 }) + expect( + await database.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, + [input.identity.userId] + ) + ).toEqual([{ count: '2' }]) + }) + + it('fails a competing aggregate supersession instead of reporting the wrong target', async () => { + const input = await fixture() + const alternate = { + id: `recovery-cell-${sequence}-alternate`, + url: `https://recovery-${sequence}-alternate.example.com`, + capacityRequests: 20 + } + await input.store.reconcileCells([...Object.values(input.cells), alternate], false) + await input.heartbeat(alternate) + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.heartbeat(alternate) + await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation) + + const results = await Promise.allSettled([ + input.store.supersedeRegisteredCellEvacuations( + input.cells.source.id, + input.cells.failed.id, + input.cells.replacement.id, + 100 + ), + input.store.supersedeRegisteredCellEvacuations( + input.cells.source.id, + input.cells.failed.id, + alternate.id, + 100 + ) + ]) + const fulfilled = results.filter( + (result): result is PromiseFulfilledResult => result.status === 'fulfilled' + ) + const rejected = results.filter( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + expect(fulfilled).toHaveLength(1) + expect(fulfilled[0]!.value).toBe(1) + expect(rejected).toHaveLength(1) + expect(rejected[0]!.reason).toMatchObject({ message: 'migration_already_superseded' }) + const successor = await database.query( + `SELECT assignment.cell_id, migration.target_cell_id + FROM relay_assignments assignment + JOIN relay_assignment_migrations migration + ON migration.user_id = assignment.user_id + AND migration.relay_host_id = assignment.relay_host_id + AND migration.assignment_epoch = assignment.assignment_epoch + WHERE assignment.user_id = ? AND migration.previous_epoch = ?`, + [input.identity.userId, input.migration.assignmentEpoch] + ) + expect(successor).toHaveLength(1) + expect(successor[0]!.cell_id).toBe(successor[0]!.target_cell_id) + expect([input.cells.replacement.id, alternate.id]).toContain( + successor[0]!.target_cell_id + ) + }) + + it('rolls back every supersession change when replacement capacity is insufficient', async () => { + const input = await fixture(1) + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence( + input.cells.failed.id, + input.cellIncarnation + ) + + await expect( + input.store.supersedeRegisteredEvacuation(input.identity, { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + currentTargetCellId: input.cells.failed.id, + replacementTargetCellId: input.cells.replacement.id + }) + ).rejects.toThrow('relay_capacity_exhausted') + expect(await reservations(input.cells)).toEqual({ source: 1, failed: 2, replacement: 0 }) + expect( + await database.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`, + [input.identity.userId] + ) + ).toEqual([{ cell_id: input.cells.failed.id, assignment_epoch: '2' }]) + expect( + await database.query( + `SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations + WHERE user_id = ?`, + [input.identity.userId] + ) + ).toEqual([{ assignment_epoch: '2', aborted_at: null }]) + }) + + it('rolls back supersession when migration request-unit shape drifted', async () => { + const input = await fixture() + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation) + await database.query( + `UPDATE relay_assignment_activity_leases + SET request_units = request_units + 1 + WHERE user_id = ? AND activity_kind = 'migration'`, + [input.identity.userId] + ) + await database.query( + `UPDATE relay_cells SET reserved_requests = reserved_requests + 1 WHERE cell_id = ?`, + [input.cells.failed.id] + ) + + await expect( + input.store.supersedeRegisteredEvacuation(input.identity, { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + currentTargetCellId: input.cells.failed.id, + replacementTargetCellId: input.cells.replacement.id + }) + ).rejects.toThrow('migration_activity_lease_shape_mismatch') + expect(await reservations(input.cells)).toEqual({ source: 1, failed: 3, replacement: 0 }) + expect( + await database.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`, + [input.identity.userId] + ) + ).toEqual([{ cell_id: input.cells.failed.id, assignment_epoch: '2' }]) + }) + + it('rolls back supersession when locked cell reservation accounting drifted', async () => { + const input = await fixture() + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation) + await database.query( + `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = ?`, + [input.cells.replacement.id] + ) + + await expect( + input.store.supersedeRegisteredEvacuation(input.identity, { + assignmentEpoch: input.migration.assignmentEpoch, + sourceCellId: input.cells.source.id, + currentTargetCellId: input.cells.failed.id, + replacementTargetCellId: input.cells.replacement.id + }) + ).rejects.toThrow('migration_cell_reservation_accounting_mismatch') + expect(await reservations(input.cells)).toEqual({ source: 1, failed: 2, replacement: 1 }) + expect( + await database.query( + `SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations + WHERE user_id = ?`, + [input.identity.userId] + ) + ).toEqual([{ assignment_epoch: '2', aborted_at: null }]) + }) + + it('rejects supersession before incrementing the maximum safe epoch', async () => { + const input = await fixture() + await expect( + input.store.supersedeRegisteredEvacuation(input.identity, { + assignmentEpoch: Number.MAX_SAFE_INTEGER, + sourceCellId: input.cells.source.id, + currentTargetCellId: input.cells.failed.id, + replacementTargetCellId: input.cells.replacement.id + }) + ).rejects.toThrow('assignment_epoch_exhausted') + }) + + async function fenceFailedCell(input: RecoveryFixture): Promise { + await input.store.setCellEnabled(input.cells.failed.id, false) + input.advancePastHeartbeat() + await input.heartbeat(input.cells.source) + await input.heartbeat(input.cells.replacement) + await input.store.attestCellFence( + input.cells.failed.id, + input.cellIncarnation + ) + } + + async function supersedeAggregate(input: RecoveryFixture): Promise { + return await input.store.supersedeRegisteredCellEvacuations( + input.cells.source.id, + input.cells.failed.id, + input.cells.replacement.id, + 100 + ) + } + + async function clearActivities(input: RecoveryFixture): Promise { + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + await database.query( + `UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0, + reserved_invites = 0, pending_installs = 0, pending_confirmations = 0, + migration_leases = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [input.identity.userId, input.identity.relayHostId] + ) + await database.query( + `UPDATE relay_cells SET reserved_requests = 0 + WHERE cell_id IN (?, ?, ?)`, + [input.cells.source.id, input.cells.failed.id, input.cells.replacement.id] + ) + } + + async function pinMigration( + input: RecoveryFixture, + assignmentEpoch: number + ): Promise { + const drainAttemptId = `${input.identity.userId}-drain-${assignmentEpoch}` + await database.query( + `INSERT INTO relay_post_drain_migration_pins + (user_id, relay_host_id, assignment_epoch, drain_attempt_id, + source_cell_id, source_cell_incarnation, target_cell_id, + target_cell_incarnation, source_request_units, target_reserved_units, + pinned_at) + SELECT migration.user_id, migration.relay_host_id, + migration.assignment_epoch, ?, migration.source_cell_id, + incarnation.source_cell_incarnation, migration.target_cell_id, + incarnation.target_cell_incarnation, migration.source_request_units, + migration.target_reserved_units, 100 + FROM relay_assignment_migrations migration + JOIN relay_assignment_migration_incarnations incarnation + ON incarnation.user_id = migration.user_id + AND incarnation.relay_host_id = migration.relay_host_id + AND incarnation.assignment_epoch = migration.assignment_epoch + WHERE migration.user_id = ? AND migration.relay_host_id = ? + AND migration.assignment_epoch = ?`, + [ + drainAttemptId, + input.identity.userId, + input.identity.relayHostId, + assignmentEpoch + ] + ) + return drainAttemptId + } + + async function assignmentActivities(input: RecoveryFixture): Promise { + return await database.query( + `SELECT activity_id, activity_kind, cell_id, request_units + FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`, + [input.identity.userId, input.identity.relayHostId] + ) + } + + async function reservations(cells: RecoveryFixture['cells']): Promise> { + const rows = await database.query( + `SELECT cell_id, reserved_requests FROM relay_cells + WHERE cell_id IN (?, ?, ?) ORDER BY cell_id`, + [cells.source.id, cells.failed.id, cells.replacement.id] + ) + return Object.fromEntries( + rows.map((row) => { + const cellId = String(row.cell_id) + const name = cellId.endsWith('-source') + ? 'source' + : cellId.endsWith('-failed') + ? 'failed' + : 'replacement' + return [name, Number(row.reserved_requests)] + }) + ) + } +}) diff --git a/cloud/apps/relay/src/observed-relay-database.ts b/cloud/apps/relay/src/observed-relay-database.ts new file mode 100644 index 00000000000..4720cae6c98 --- /dev/null +++ b/cloud/apps/relay/src/observed-relay-database.ts @@ -0,0 +1,46 @@ +import type { + RelayDatabase, + RelayLockOptions, + RelayTransactionOptions, + SqlRow +} from './database.js' +import { timedRelayOperation, type RelayRuntimeObserver } from './relay-observability.js' + +export function observeRelayDatabase( + database: RelayDatabase, + observer: RelayRuntimeObserver +): RelayDatabase { + const query = (sql: string, params?: unknown[]): Promise => + timedRelayOperation( + () => database.query(sql, params), + (durationMs, success) => observer.recordSql(durationMs, success) + ) + const queryLocked = ( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise => + timedRelayOperation( + () => database.queryLocked(sql, params, options), + (durationMs, success) => observer.recordSql(durationMs, success), + (error) => + // NOWAIT contention is an intentional sweep deferral, not a SQL-health failure. + options?.failIfUnavailable === true && + error instanceof Error && + error.message === 'database_lock_unavailable' + ) + return { + dialect: database.dialect, + query, + queryLocked, + transaction: async ( + operation: (transaction: RelayDatabase) => Promise, + options?: RelayTransactionOptions + ): Promise => + await database.transaction( + async (transaction) => await operation(observeRelayDatabase(transaction, observer)), + options + ), + close: async () => await database.close() + } +} diff --git a/cloud/apps/relay/src/postgres-drain-send-locking.test.ts b/cloud/apps/relay/src/postgres-drain-send-locking.test.ts new file mode 100644 index 00000000000..024bf46db5e --- /dev/null +++ b/cloud/apps/relay/src/postgres-drain-send-locking.test.ts @@ -0,0 +1,215 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const source = { + id: 'drain-send-lock-source', + url: 'https://drain-send-lock-source.example.com', + capacityRequests: 100 +} +const target = { + id: 'drain-send-lock-target', + url: 'https://drain-send-lock-target.example.com', + capacityRequests: 100 +} +const identity = { + userId: 'drain-send-lock-user', + relayHostId: 'drainsendlock01' +} +const sourceIncarnation = '11111111-1111-4111-8111-111111111111' +const targetIncarnation = '22222222-2222-4222-8222-222222222222' +const attemptId = '33333333-3333-4333-8333-333333333333' + +describePostgres('PostgreSQL drain-send locking', () => { + let database: RelayDatabase + + beforeAll(async () => { + database = await openRelayDatabase({ databaseUrl, dataDir: '' }) + }) + + afterAll(async () => await database.close()) + + afterEach(async () => { + await database.query(`DELETE FROM relay_post_drain_migration_pins WHERE user_id = ?`, [ + identity.userId + ]) + await database.query(`DELETE FROM relay_cell_drain_attempt_states WHERE attempt_id = ?`, [ + attemptId + ]) + await database.query( + `DELETE FROM relay_control_connection_reservations WHERE user_id = ?`, + [identity.userId] + ) + await database.query(`DELETE FROM relay_migration_leases WHERE user_id = ?`, [ + identity.userId + ]) + await database.query(`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, [ + identity.userId + ]) + await database.query( + `DELETE FROM relay_assignment_migration_incarnations WHERE user_id = ?`, + [identity.userId] + ) + await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [ + identity.userId + ]) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId]) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id IN (?, ?)`, [ + source.id, + target.id + ]) + await database.query(`DELETE FROM relay_cell_admission WHERE cell_id IN (?, ?)`, [ + source.id, + target.id + ]) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [ + source.id, + target.id + ]) + }) + + it('locks active migrations without locking the nullable incarnation lookup', async () => { + const now = 1_700_000_000_000 + const store = new RelayAssignmentStore(database, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells([source, target]) + await store.recordCellHeartbeat({ + cellId: source.id, + cellUrl: source.url, + cellIncarnation: sourceIncarnation, + startedAt: now - 1, + ready: true, + observedRequests: 0 + }) + await store.recordCellHeartbeat({ + cellId: target.id, + cellUrl: target.url, + cellIncarnation: targetIncarnation, + startedAt: now - 1, + ready: true, + observedRequests: 0 + }) + await store.assign(identity) + await store.setCellEnabled(source.id, false) + await store.startEvacuation(identity, target.id) + await store.prepareCellDrainAttempt({ + attemptId, + cellId: source.id, + cellIncarnation: sourceIncarnation, + traceValue: '44444444-4444-4444-8444-444444444444', + plannedGraceMs: 120_000 + }) + await expect( + store.prepareCellDrainRecovery({ + attemptId, + cellId: source.id, + cellIncarnation: sourceIncarnation + }) + ).resolves.toMatchObject({ + shouldSend: false, + preparedAttempt: { attemptId, state: 'prepared' } + }) + + await expect( + store.beginCellDrainSend({ + attemptId, + cellId: source.id, + cellIncarnation: sourceIncarnation + }) + ).resolves.toMatchObject({ state: 'send-may-have-started', shouldSend: true }) + await expect( + store.beginCellDrainSend({ + attemptId, + cellId: source.id, + cellIncarnation: sourceIncarnation + }) + ).resolves.toMatchObject({ state: 'send-may-have-started', shouldSend: false }) + await expect( + store.prepareCellDrainRecovery({ + attemptId, + cellId: source.id, + cellIncarnation: sourceIncarnation + }) + ).rejects.toThrow('drain_application_receipt_missing') + await expect( + database.query(`SELECT drain_attempt_id FROM relay_post_drain_migration_pins`) + ).resolves.toEqual([{ drain_attempt_id: attemptId }]) + }) + + it('restores an expired registered migration lease with PostgreSQL locks', async () => { + let now = 1_700_000_000_000 + const startedAt = now - 1 + const store = new RelayAssignmentStore(database, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells([source, target]) + await store.recordCellHeartbeat({ + cellId: source.id, + cellUrl: source.url, + cellIncarnation: sourceIncarnation, + startedAt, + ready: true, + observedRequests: 0 + }) + await store.recordCellHeartbeat({ + cellId: target.id, + cellUrl: target.url, + cellIncarnation: targetIncarnation, + startedAt, + ready: true, + observedRequests: 0 + }) + await store.assign(identity) + await store.setCellEnabled(source.id, false) + const migration = await store.startEvacuation(identity, target.id) + await store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: migration.assignmentEpoch + }) + await store.prepareCellDrainAttempt({ + attemptId, + cellId: source.id, + cellIncarnation: sourceIncarnation, + traceValue: '44444444-4444-4444-8444-444444444444', + plannedGraceMs: 120_000 + }) + + now = migration.expiresAt + 1 + expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0) + await store.recordCellHeartbeat({ + cellId: source.id, + cellUrl: source.url, + cellIncarnation: sourceIncarnation, + startedAt, + ready: true, + observedRequests: 0 + }) + await store.recordCellHeartbeat({ + cellId: target.id, + cellUrl: target.url, + cellIncarnation: targetIncarnation, + startedAt, + ready: true, + observedRequests: 0 + }) + await expect( + store.beginCellDrainSend({ + attemptId, + cellId: source.id, + cellIncarnation: sourceIncarnation + }) + ).resolves.toMatchObject({ state: 'send-may-have-started', shouldSend: true }) + await expect( + database.query( + `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).resolves.toContainEqual({ activity_kind: 'migration', cell_id: target.id }) + }) +}) diff --git a/cloud/apps/relay/src/postgres-idle-client-error.test.ts b/cloud/apps/relay/src/postgres-idle-client-error.test.ts new file mode 100644 index 00000000000..50a7bb48ab5 --- /dev/null +++ b/cloud/apps/relay/src/postgres-idle-client-error.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it, vi } from 'vitest' +import { absorbPostgresIdleClientErrors } from './database.js' + +describe('PostgreSQL idle-client failure handling', () => { + it('absorbs the pool error without logging connection details', () => { + let listener: ((error: Error) => void) | undefined + const pool = { + on: vi.fn((_event: string, value: (error: Error) => void) => { + listener = value + return pool + }) + } + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + absorbPostgresIdleClientErrors(pool as never) + expect(() => listener?.(new Error('postgres://user:secret@database'))).not.toThrow() + expect(warning).toHaveBeenCalledWith('[orca-relay] idle PostgreSQL client failed') + expect(JSON.stringify(warning.mock.calls)).not.toContain('secret') + + warning.mockRestore() + }) +}) diff --git a/cloud/apps/relay/src/postgres-maintenance-sweep-plans.test.ts b/cloud/apps/relay/src/postgres-maintenance-sweep-plans.test.ts new file mode 100644 index 00000000000..a8d0e1e3bf9 --- /dev/null +++ b/cloud/apps/relay/src/postgres-maintenance-sweep-plans.test.ts @@ -0,0 +1,61 @@ +import pg from 'pg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip + +// Inactive bases outlive their sweep and are never pruned, so the table only +// grows; production reached ~2.24M rows of which 3 were active. Enough rows +// here that a sequential scan is the cheaper plan without the index. +const INACTIVE_ROWS = 20_000 +const OWNED_PREFIX = 'sweep-plan-' + +describePostgres('PostgreSQL maintenance sweep plans', () => { + let database: RelayDatabase + let client: pg.Client + + beforeAll(async () => { + database = await openRelayDatabase({ databaseUrl, dataDir: '' }) + // Only ever touch this suite's own rows: the database is shared with the + // other PostgreSQL suites running in parallel. + await database.query( + `DELETE FROM relay_connection_bases WHERE basis_conn_id LIKE ?`, + [`${OWNED_PREFIX}%`] + ) + await database.query( + `INSERT INTO relay_connection_bases + (basis_conn_id, user_id, relay_host_id, relay_device_id, + owning_control_generation, credential_kind, deadline, active, created_at) + SELECT '${OWNED_PREFIX}' || generation, 'sweep-plan-user', 'sweepplan01', + 'sweep-plan-device', 1, 'invite', 1000, 0, 1000 + FROM generate_series(1, ${INACTIVE_ROWS}) AS generation` + ) + await database.query(`ANALYZE relay_connection_bases`) + client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + }) + + afterAll(async () => { + await client.end() + await database.query( + `DELETE FROM relay_connection_bases WHERE basis_conn_id LIKE ?`, + [`${OWNED_PREFIX}%`] + ) + await database.close() + }) + + // Why: a seq scan here held the maintenance transaction open long enough to + // time out assignment lock waits fleet-wide (2026-08-05 incident). + it('matches expired active bases by index instead of scanning the table', async () => { + const result = await client.query( + `EXPLAIN UPDATE relay_connection_bases SET active = $1 + WHERE active = $2 AND deadline <= $3`, + [0, 1, 2000] + ) + const plan = result.rows.map((row) => String(row['QUERY PLAN'])).join('\n') + + expect(plan).not.toMatch(/Seq Scan on relay_connection_bases/) + expect(plan).toMatch(/relay_connection_bases_active_deadline/) + }) +}) diff --git a/cloud/apps/relay/src/postgres-pool-pressure.test.ts b/cloud/apps/relay/src/postgres-pool-pressure.test.ts new file mode 100644 index 00000000000..2e020bf43fb --- /dev/null +++ b/cloud/apps/relay/src/postgres-pool-pressure.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it, vi } from 'vitest' +import { PostgresPoolPressure } from './postgres-pool-pressure.js' + +describe('PostgreSQL pool pressure', () => { + it('reports current waiters and interval high-water marks', async () => { + let now = 1_000 + let resolveConnection!: (client: unknown) => void + const connection = new Promise((resolve) => { + resolveConnection = resolve + }) + const pool = { + totalCount: 3, + idleCount: 0, + waitingCount: 0, + connect: vi.fn(() => { + pool.waitingCount++ + return connection + }) + } + const pressure = new PostgresPoolPressure(pool as never, () => now) + const pending = pressure.connect() + now = 1_750 + + expect(pressure.consumeCounts()).toMatchObject({ + databasePoolTotal: 3, + databasePoolIdle: 0, + databasePoolWaiting: 1, + databasePoolWaitersMax: 1, + databasePoolOldestWaitMs: 750, + databasePoolWaitMsMax: 750 + }) + + now = 2_250 + pool.waitingCount-- + resolveConnection({ query: vi.fn(), release: vi.fn() }) + await pending + expect(pressure.consumeCounts()).toMatchObject({ + databasePoolWaiting: 0, + databasePoolWaitersMax: 1, + databasePoolOldestWaitMs: 0, + databasePoolWaitMsMax: 1_250 + }) + expect(pressure.consumeCounts()).toMatchObject({ + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + }) +}) diff --git a/cloud/apps/relay/src/postgres-pool-pressure.ts b/cloud/apps/relay/src/postgres-pool-pressure.ts new file mode 100644 index 00000000000..e18bf2bdd42 --- /dev/null +++ b/cloud/apps/relay/src/postgres-pool-pressure.ts @@ -0,0 +1,93 @@ +import type pg from 'pg' + +export type PostgresPoolPressureCounts = { + databasePoolTotal: number + databasePoolIdle: number + databasePoolWaiting: number + databasePoolWaitersMax: number + databasePoolOldestWaitMs: number + databasePoolWaitMsMax: number +} + +const emptyCounts = (): PostgresPoolPressureCounts => ({ + databasePoolTotal: 0, + databasePoolIdle: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolOldestWaitMs: 0, + databasePoolWaitMsMax: 0 +}) + +export class PostgresPoolPressure { + private readonly waiters = new Map() + private waitersMax = 0 + private waitMsMax = 0 + private lastConsumed = emptyCounts() + + constructor( + private readonly pool: pg.Pool, + private readonly now: () => number = Date.now + ) {} + + async connect(): Promise { + const waitingBefore = this.pool.waitingCount + const connection = this.pool.connect() + if (this.pool.waitingCount <= waitingBefore) return await connection + + const waiter = Symbol() + const startedAt = this.now() + this.waiters.set(waiter, startedAt) + this.waitersMax = Math.max(this.waitersMax, this.waiters.size) + try { + return await connection + } finally { + this.waitMsMax = Math.max(this.waitMsMax, this.now() - startedAt) + this.waiters.delete(waiter) + } + } + + consumeCounts(): PostgresPoolPressureCounts { + const counts = this.readCounts() + this.lastConsumed = counts + this.waitersMax = this.waiters.size + this.waitMsMax = counts.databasePoolOldestWaitMs + return counts + } + + peekCounts(): PostgresPoolPressureCounts { + const current = this.readCounts() + return { + ...current, + databasePoolWaitersMax: Math.max( + current.databasePoolWaitersMax, + this.lastConsumed.databasePoolWaitersMax + ), + databasePoolOldestWaitMs: Math.max( + current.databasePoolOldestWaitMs, + this.lastConsumed.databasePoolOldestWaitMs + ), + databasePoolWaitMsMax: Math.max( + current.databasePoolWaitMsMax, + this.lastConsumed.databasePoolWaitMsMax + ) + } + } + + private readCounts(): PostgresPoolPressureCounts { + const now = this.now() + const oldestWaitMs = + this.waiters.size === 0 ? 0 : Math.max(0, now - Math.min(...this.waiters.values())) + return { + databasePoolTotal: this.pool.totalCount, + databasePoolIdle: this.pool.idleCount, + databasePoolWaiting: this.waiters.size, + databasePoolWaitersMax: Math.max(this.waitersMax, this.waiters.size), + databasePoolOldestWaitMs: oldestWaitMs, + databasePoolWaitMsMax: Math.max(this.waitMsMax, oldestWaitMs) + } + } +} + +export function emptyPostgresPoolPressureCounts(): PostgresPoolPressureCounts { + return emptyCounts() +} diff --git a/cloud/apps/relay/src/postgres-schema-concurrency-postgres.test.ts b/cloud/apps/relay/src/postgres-schema-concurrency-postgres.test.ts new file mode 100644 index 00000000000..5208f137ae8 --- /dev/null +++ b/cloud/apps/relay/src/postgres-schema-concurrency-postgres.test.ts @@ -0,0 +1,55 @@ +import pg from 'pg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const schema = 'relay_schema_concurrency_test' + +describePostgres('PostgreSQL schema concurrency', () => { + let scopedUrl = '' + + beforeAll(async () => { + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + await client.query(`CREATE SCHEMA ${schema}`) + } finally { + await client.end() + } + const url = new URL(databaseUrl!) + url.searchParams.set('options', `-c search_path=${schema}`) + scopedUrl = url.toString() + }) + + afterAll(async () => { + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + } finally { + await client.end() + } + }) + + it('opens five directors when one new table is absent', async () => { + const initial = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' }) + await initial.query(`DROP TABLE relay_cell_legacy_fence_adoptions`) + await initial.close() + + const results = await Promise.allSettled( + Array.from({ length: 5 }, async (): Promise => + await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' }) + ) + ) + const databases = results.flatMap((result) => + result.status === 'fulfilled' ? [result.value] : [] + ) + try { + expect(results.every((result) => result.status === 'fulfilled')).toBe(true) + } finally { + await Promise.all(databases.map(async (database) => await database.close())) + } + }) +}) diff --git a/cloud/apps/relay/src/postgres-schema-startup.ts b/cloud/apps/relay/src/postgres-schema-startup.ts new file mode 100644 index 00000000000..5e6260ad2fb --- /dev/null +++ b/cloud/apps/relay/src/postgres-schema-startup.ts @@ -0,0 +1,83 @@ +const RETRYABLE_SCHEMA_CODES = new Set(['55P03', '57014']) +const DEFAULT_RETRY_DEADLINE_MS = 30_000 +const RETRY_BASE_DELAY_MS = 250 +const RETRY_MAX_DELAY_MS = 2_000 + +type SchemaStartupOptions = { + now?: () => number + random?: () => number + retryDeadlineMs?: number + wait?: (delayMs: number) => Promise +} + +function retryDelayMs(attempt: number, random: () => number): number { + const ceiling = Math.min( + RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), + RETRY_MAX_DELAY_MS + ) + return Math.ceil(ceiling * (0.5 + random() * 0.5)) +} + +function wait(delayMs: number): Promise { + return new Promise((resolve) => setTimeout(resolve, delayMs)) +} + +function retryableSchemaError(error: unknown, statement: string): boolean { + const value = error as { code?: unknown; constraint?: unknown } + return ( + RETRYABLE_SCHEMA_CODES.has(String(value.code)) || + (value.code === '23505' && + ((value.constraint === 'pg_type_typname_nsp_index' && + /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i.test(statement)) || + (value.constraint === 'pg_class_relname_nsp_index' && + /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i.test(statement)))) + ) +} + +export async function applyPostgresSchema( + statements: string[], + query: (statement: string) => Promise, + options: SchemaStartupOptions = {} +): Promise { + const now = options.now ?? Date.now + const random = options.random ?? Math.random + const pause = options.wait ?? wait + const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS) + + for (const statement of statements) { + let attempt = 1 + while (true) { + try { + await query(statement) + break + } catch (error) { + const code = String((error as { code?: unknown }).code) + const remainingMs = deadlineAt - now() + const retryable = retryableSchemaError(error, statement) + if (!retryable || remainingMs <= 0) { + if (retryable) { + console.warn( + JSON.stringify({ + event: 'orca_relay_postgres_schema_retry_exhausted', + code, + attempts: attempt + }) + ) + } + throw error + } + const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random)) + console.warn( + JSON.stringify({ + event: 'orca_relay_postgres_schema_retry', + code, + attempt, + delayMs + }) + ) + await pause(delayMs) + attempt += 1 + } + } + } +} diff --git a/cloud/apps/relay/src/postgres-transaction-recovery.test.ts b/cloud/apps/relay/src/postgres-transaction-recovery.test.ts new file mode 100644 index 00000000000..a49c07d2e7d --- /dev/null +++ b/cloud/apps/relay/src/postgres-transaction-recovery.test.ts @@ -0,0 +1,1446 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { + openRelayDatabase, + type RelayDatabase, + type RelayLockOptions, + type SqlRow +} from './database.js' + +function postgresTestDatabaseUrl(value: string | undefined): string | undefined { + if (!value) return undefined + const url = new URL(value) + // Detect the intended deadlock before the one-second runtime lock deadline. + url.searchParams.set('options', '-c deadlock_timeout=100ms') + return url.toString() +} + +const databaseUrl = postgresTestDatabaseUrl(process.env.ORCA_RELAY_TEST_POSTGRES_URL) +const describePostgres = databaseUrl ? describe : describe.skip + +type QueryLockHook = (phase: 'before' | 'after', sql: string) => Promise + +class TransactionProbeDatabase implements RelayDatabase { + attempts = 0 + + constructor( + private readonly database: RelayDatabase, + private readonly hook: QueryLockHook, + private readonly probeQueries = false + ) {} + + async query(sql: string, params?: unknown[]): Promise { + return await this.database.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + return await this.database.queryLocked(sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await this.database.transaction(async (transaction) => { + this.attempts++ + return await operation( + new QueryLockProbeTransaction(transaction, this.hook, this.probeQueries) + ) + }) + } + + async close(): Promise {} +} + +class QueryLockProbeTransaction implements RelayDatabase { + constructor( + private readonly transactionDatabase: RelayDatabase, + private readonly hook: QueryLockHook, + private readonly probeQueries: boolean + ) {} + + async query(sql: string, params?: unknown[]): Promise { + if (this.probeQueries) await this.hook('before', sql) + const rows = await this.transactionDatabase.query(sql, params) + if (this.probeQueries) await this.hook('after', sql) + return rows + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + await this.hook('before', sql) + const rows = await this.transactionDatabase.queryLocked(sql, params, options) + await this.hook('after', sql) + return rows + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await operation(this) + } + + async close(): Promise {} +} + +function signal(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + return { promise: new Promise((done) => (resolve = done)), resolve } +} + +describePostgres('PostgreSQL transaction recovery', () => { + let database: RelayDatabase + + beforeAll(async () => { + database = await openRelayDatabase({ databaseUrl, dataDir: '' }) + }) + + afterAll(async () => { + await database.close() + }) + + afterEach(async () => { + for (const identity of [ + { userId: 'released-order-user', relayHostId: 'releasedorder001' }, + { userId: 'normalized-order-user', relayHostId: 'normalizedorder1' }, + { userId: 'atomic-final-user-a', relayHostId: 'atomicfinalhosta' }, + { userId: 'atomic-final-user-b', relayHostId: 'atomicfinalhostb' }, + { userId: 'lease-assignment-contention-user', relayHostId: 'leaseassignment1' } + ]) { + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database.query( + `DELETE FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + } + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [ + 'released-order-cell', + 'normalized-order-cell' + ]) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, ['atomic-final-cell']) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [ + 'lease-assignment-contention-cell' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'reconcile-user-%'` + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'reconcile-user-%'`) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [ + 'reconcile-cell-a', + 'reconcile-cell-b' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, + ['completion-race-user'] + ) + await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [ + 'completion-race-user' + ]) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [ + 'completion-race-user' + ]) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [ + 'completion-race-source', + 'completion-race-target' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, + ['completion-contention-user'] + ) + await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [ + 'completion-contention-user' + ]) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [ + 'completion-contention-user' + ]) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id IN (?, ?)`, [ + 'completion-contention-source', + 'completion-contention-target' + ]) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [ + 'completion-contention-source', + 'completion-contention-target' + ]) + await database.query(`DELETE FROM relay_cell_fences WHERE cell_id = ?`, [ + 'dead-source-contention-source' + ]) + await database.query( + `DELETE FROM relay_control_connection_reservations WHERE user_id = ?`, + ['dead-source-contention-user'] + ) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, + ['dead-source-contention-user'] + ) + await database.query( + `DELETE FROM relay_assignment_migration_incarnations WHERE user_id = ?`, + ['dead-source-contention-user'] + ) + await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [ + 'dead-source-contention-user' + ]) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [ + 'dead-source-contention-user' + ]) + await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id IN (?, ?)`, [ + 'dead-source-contention-source', + 'dead-source-contention-target' + ]) + await database.query(`DELETE FROM relay_cell_admission WHERE cell_id IN (?, ?)`, [ + 'dead-source-contention-source', + 'dead-source-contention-target' + ]) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [ + 'dead-source-contention-source', + 'dead-source-contention-target' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id IN (?, ?)`, + ['lease-contention-user', 'aggregate-contention-user'] + ) + await database.query( + `DELETE FROM relay_assignments + WHERE user_id IN (?, ?)`, + ['lease-contention-user', 'aggregate-contention-user'] + ) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [ + 'lease-contention-cell', + 'aggregate-contention-cell' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, + ['sustained-lock-user'] + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [ + 'sustained-lock-user' + ]) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [ + 'sustained-lock-cell' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, + ['sticky-cell-contention-user'] + ) + await database.query( + `DELETE FROM relay_assignments WHERE user_id = ?`, + ['sticky-cell-contention-user'] + ) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [ + 'sticky-cell-contention-cell' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'postgres-user-%'` + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'postgres-user-%'`) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?, ?)`, [ + 'cell-a', + 'cell-b', + 'cell-c' + ]) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'sticky-fast-user-%'` + ) + await database.query( + `DELETE FROM relay_assignments WHERE user_id LIKE 'sticky-fast-user-%'` + ) + await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [ + 'sticky-fast-cell' + ]) + }) + + it('retries an entire deadlock victim transaction on a fresh attempt', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const keys = ['deadlock-a', 'deadlock-b'] + for (const key of keys) { + await database.query( + `INSERT INTO relay_rate_windows + (scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?) + ON CONFLICT (scope_key, window_kind, window_started_at) DO UPDATE SET count = ?`, + [key, 'transaction-recovery', 1, 0, 0] + ) + } + + let firstAttemptArrivals = 0 + let releaseFirstAttempts!: () => void + const firstAttemptsReady = new Promise((resolve) => { + releaseFirstAttempts = resolve + }) + const attempts = [0, 0] + const mutateWithOppositeLockOrder = async ( + operationIndex: number, + firstKey: string, + secondKey: string + ): Promise => { + await database.transaction(async (transaction) => { + attempts[operationIndex] = (attempts[operationIndex] ?? 0) + 1 + await transaction.queryLocked( + `SELECT * FROM relay_rate_windows + WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`, + [firstKey, 'transaction-recovery', 1] + ) + if (attempts[operationIndex] === 1) { + firstAttemptArrivals++ + if (firstAttemptArrivals === 2) releaseFirstAttempts() + await firstAttemptsReady + } + await transaction.queryLocked( + `SELECT * FROM relay_rate_windows + WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`, + [secondKey, 'transaction-recovery', 1] + ) + await transaction.query( + `UPDATE relay_rate_windows SET count = count + 1 + WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`, + [firstKey, 'transaction-recovery', 1] + ) + }) + } + + await Promise.all([ + mutateWithOppositeLockOrder(0, keys[0]!, keys[1]!), + mutateWithOppositeLockOrder(1, keys[1]!, keys[0]!) + ]) + + expect([...attempts].sort()).toEqual([1, 2]) + const rows = await database.query( + `SELECT scope_key, count FROM relay_rate_windows + WHERE window_kind = ? ORDER BY scope_key`, + ['transaction-recovery'] + ) + expect(rows).toEqual([ + { scope_key: keys[0], count: '1' }, + { scope_key: keys[1], count: '1' } + ]) + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('"event":"orca_relay_postgres_transaction_retry"') + ) + expect(warn).toHaveBeenCalledWith(expect.stringContaining('"phase":"rate-limit"')) + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('"event":"orca_relay_postgres_transaction_exhausted"') + ) + warn.mockRestore() + }, 10_000) + + it('defers assignment around a released-cell activity transaction', async () => { + const now = 1_100_000_000_000 + const identity = { userId: 'released-order-user', relayHostId: 'releasedorder001' } + const cellId = 'released-order-cell' + const activityId = 'splice:released-order' + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, [ + identity.userId, + identity.relayHostId + ]) + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells([ + { id: cellId, url: 'https://released-order.example.com', capacityRequests: 100 } + ]) + const assignment = await seedStore.assign(identity) + await seedStore.acquireActivity(identity, { activityId, kind: 'splice', cellId }) + + const assignmentLocked = signal() + const legacyCellLocked = signal() + let gateFirstAssignmentAttempt = true + const directorDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if ( + gateFirstAssignmentAttempt && + phase === 'after' && + sql.includes('FROM relay_assignments') + ) { + gateFirstAssignmentAttempt = false + assignmentLocked.resolve() + await legacyCellLocked.promise + } + }) + const directorStore = new RelayAssignmentStore(directorDatabase, () => now) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + const directorAssign = directorStore.assign(identity) + await assignmentLocked.promise + let legacyAttempts = 0 + const legacyRelease = database.transaction(async (transaction) => { + legacyAttempts++ + const lease = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, activityId] + ) + )[0] + if (!lease) return + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId]) + legacyCellLocked.resolve() + await transaction.query( + `UPDATE relay_cells SET reserved_requests = reserved_requests - 2 WHERE cell_id = ?`, + [cellId] + ) + await transaction.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`, + [identity.userId, identity.relayHostId, activityId] + ) + await transaction.query( + `UPDATE relay_assignments SET reserved_splices = reserved_splices - 1 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + }) + + await Promise.all([directorAssign, legacyRelease]) + + expect(directorDatabase.attempts).toBe(2) + expect(legacyAttempts).toBe(1) + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('orca_relay_postgres_transaction_retry') + ) + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('orca_relay_postgres_transaction_exhausted') + ) + await expect(seedStore.resolve(identity)).resolves.toMatchObject({ + cellId, + assignmentEpoch: assignment.assignmentEpoch + }) + const state = await database.query( + `SELECT assignment.reserved_controls, assignment.reserved_splices, + cell.reserved_requests + FROM relay_assignments assignment + JOIN relay_cells cell ON cell.cell_id = assignment.cell_id + WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + expect(state).toEqual([ + { reserved_controls: '1', reserved_splices: '0', reserved_requests: '1' } + ]) + warn.mockRestore() + }, 15_000) + + it('renews an existing control without waiting on a legacy cell lock', async () => { + const now = 1_150_000_000_000 + const identity = { userId: 'sustained-lock-user', relayHostId: 'sustainedlock01' } + const cell = { + id: 'sustained-lock-cell', + url: 'https://sustained-lock.example.com', + capacityRequests: 100 + } + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells([cell]) + await seedStore.assign(identity) + + const legacyCellLocked = signal() + const releaseLegacyCell = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cell.id]) + legacyCellLocked.resolve() + await releaseLegacyCell.promise + }) + await legacyCellLocked.promise + + let inventoryAttempts = 0 + const assignmentDatabase = new TransactionProbeDatabase( + database, + async (phase, sql) => { + if (phase !== 'before' || !sql.includes('FROM relay_cells ORDER BY')) return + inventoryAttempts++ + } + ) + const assignmentStore = new RelayAssignmentStore(assignmentDatabase, () => now) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + await expect(assignmentStore.assign(identity)).resolves.toMatchObject({ + cellId: cell.id + }) + releaseLegacyCell.resolve() + await expect(legacyTransaction).resolves.toBeUndefined() + expect(inventoryAttempts).toBe(0) + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('orca_relay_postgres_transaction_retry') + ) + warn.mockRestore() + }, 15_000) + + it('retries a new sticky control without forming a legacy cell-first deadlock', async () => { + const now = 1_175_000_000_000 + const identity = { + userId: 'sticky-cell-contention-user', + relayHostId: 'stickycellwait1' + } + const cell = { + id: 'sticky-cell-contention-cell', + url: 'https://sticky-cell-contention.example.com', + capacityRequests: 100 + } + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells([cell]) + await seedStore.assign(identity) + await seedStore.changeActivity(identity, 'migration', 1) + // Drops the grant's pending lease too: a sticky control the rows do not + // show is what sends the retry through the NOWAIT cell lock. + await seedStore.releaseActivity(identity, 'control-pending:1') + + const legacyCellLocked = signal() + const directorAssignmentLocked = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cell.id]) + legacyCellLocked.resolve() + await directorAssignmentLocked.promise + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + }) + await legacyCellLocked.promise + + let signalFirstAttempt = true + const directorLockOrder: string[] = [] + const assignmentDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if (phase === 'before') { + if (sql.includes('FROM relay_assignments WHERE user_id = ?')) { + directorLockOrder.push('assignment') + } else if (sql.includes('FROM relay_assignment_activity_leases')) { + directorLockOrder.push('activity') + } else if (sql.includes('FROM relay_cells ORDER BY')) { + directorLockOrder.push('cell-inventory') + } else if (sql.includes('FROM relay_cells WHERE cell_id = ?')) { + directorLockOrder.push('cell') + } + } + if ( + signalFirstAttempt && + phase === 'after' && + sql.includes('FROM relay_assignments WHERE user_id = ?') + ) { + signalFirstAttempt = false + directorAssignmentLocked.resolve() + } + }) + const store = new RelayAssignmentStore(assignmentDatabase, () => now) + + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: cell.id, + assignmentEpoch: 1 + }) + await expect(legacyTransaction).resolves.toBeUndefined() + expect(assignmentDatabase.attempts).toBe(2) + expect(directorLockOrder).toEqual([ + 'assignment', + 'activity', + 'cell', + 'cell-inventory', + 'assignment', + 'activity' + ]) + const state = await database.query( + `SELECT assignment.reserved_controls, assignment.migration_leases, + cell.reserved_requests + FROM relay_assignments assignment + JOIN relay_cells cell ON cell.cell_id = assignment.cell_id + WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + expect(state).toEqual([ + { reserved_controls: '1', migration_leases: '1', reserved_requests: '2' } + ]) + }, 15_000) + + it('serializes normalized assignment and activity release without a retry', async () => { + const now = 1_200_000_000_000 + const identity = { userId: 'normalized-order-user', relayHostId: 'normalizedorder1' } + const cellId = 'normalized-order-cell' + const activityId = 'splice:normalized-order' + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, [ + identity.userId, + identity.relayHostId + ]) + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells([ + { id: cellId, url: 'https://normalized-order.example.com', capacityRequests: 100 } + ]) + await seedStore.assign(identity) + await seedStore.acquireActivity(identity, { activityId, kind: 'splice', cellId }) + + const assignmentLocked = signal() + const releaseReachedAssignment = signal() + const continueAssignment = signal() + let gateFirstAssignmentAttempt = true + const assignDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if ( + gateFirstAssignmentAttempt && + phase === 'after' && + sql.includes('FROM relay_assignments') + ) { + gateFirstAssignmentAttempt = false + assignmentLocked.resolve() + await continueAssignment.promise + } + }) + const releaseDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if (phase === 'before' && sql.includes('FROM relay_assignments')) { + releaseReachedAssignment.resolve() + } + }) + const assignStore = new RelayAssignmentStore(assignDatabase, () => now) + const releaseStore = new RelayAssignmentStore(releaseDatabase, () => now) + + const assign = assignStore.assign(identity) + await assignmentLocked.promise + const release = releaseStore.releaseActivity(identity, activityId) + await releaseReachedAssignment.promise + continueAssignment.resolve() + await Promise.all([assign, release]) + + expect(assignDatabase.attempts).toBe(1) + expect(releaseDatabase.attempts).toBe(1) + const state = await database.query( + `SELECT assignment.reserved_controls, assignment.reserved_splices, + cell.reserved_requests + FROM relay_assignments assignment + JOIN relay_cells cell ON cell.cell_id = assignment.cell_id + WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + expect(state).toEqual([ + { reserved_controls: '1', reserved_splices: '0', reserved_requests: '1' } + ]) + }, 15_000) + + it('takes the shared cell lock only for the final atomic activity write', async () => { + const now = 1_250_000_000_000 + const cell = { + id: 'atomic-final-cell', + url: 'https://atomic-final.example.com', + capacityRequests: 4 + } + const identities = [ + { userId: 'atomic-final-user-a', relayHostId: 'atomicfinalhosta' }, + { userId: 'atomic-final-user-b', relayHostId: 'atomicfinalhostb' } + ] + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells([cell]) + await Promise.all(identities.map(async (identity) => await seedStore.assign(identity))) + + const firstAtCellWrite = signal() + const secondAtCellWrite = signal() + const releaseFirst = signal() + const releaseSecond = signal() + const isAtomicCellWrite = (sql: string): boolean => + sql.includes('UPDATE relay_cells SET reserved_requests') && + sql.includes('RETURNING cell_id') + const firstDatabase = new TransactionProbeDatabase( + database, + async (phase, sql) => { + if (phase !== 'before' || !isAtomicCellWrite(sql)) return + firstAtCellWrite.resolve() + await releaseFirst.promise + }, + true + ) + const secondDatabase = new TransactionProbeDatabase( + database, + async (phase, sql) => { + if (phase !== 'before' || !isAtomicCellWrite(sql)) return + secondAtCellWrite.resolve() + await releaseSecond.promise + }, + true + ) + const first = new RelayAssignmentStore(firstDatabase, () => now).acquireActivity( + identities[0]!, + { activityId: 'splice:atomic-final-a', kind: 'splice', cellId: cell.id } + ) + await firstAtCellWrite.promise + const second = new RelayAssignmentStore(secondDatabase, () => now).acquireActivity( + identities[1]!, + { activityId: 'splice:atomic-final-b', kind: 'splice', cellId: cell.id } + ) + let reachTimeout: ReturnType | undefined + const secondReachedCellWrite = await Promise.race([ + secondAtCellWrite.promise.then(() => true), + new Promise((resolve) => { + reachTimeout = setTimeout(() => resolve(false), 2_000) + }) + ]) + if (reachTimeout) clearTimeout(reachTimeout) + if (!secondReachedCellWrite) { + releaseFirst.resolve() + releaseSecond.resolve() + await Promise.allSettled([first, second]) + } + expect(secondReachedCellWrite).toBe(true) + + releaseFirst.resolve() + await expect(first).resolves.toBeUndefined() + releaseSecond.resolve() + await expect(second).rejects.toThrow('relay_capacity_exhausted') + await expect( + database.query( + `SELECT cell.reserved_requests, COUNT(lease.activity_id) AS leases, + COALESCE(SUM(lease.request_units), 0) AS lease_units + FROM relay_cells cell + LEFT JOIN relay_assignment_activity_leases lease ON lease.cell_id = cell.cell_id + WHERE cell.cell_id = ? GROUP BY cell.cell_id, cell.reserved_requests`, + [cell.id] + ) + ).resolves.toEqual([{ reserved_requests: '4', leases: '3', lease_units: '4' }]) + await expect( + database.query( + `SELECT user_id, reserved_splices FROM relay_assignments + WHERE user_id IN (?, ?) ORDER BY user_id`, + [identities[0]!.userId, identities[1]!.userId] + ) + ).resolves.toEqual([ + { user_id: identities[0]!.userId, reserved_splices: '1' }, + { user_id: identities[1]!.userId, reserved_splices: '0' } + ]) + }, 15_000) + + it('reconciles drift under the assignment-first lock order while activity waits', async () => { + const now = 1_300_000_000_000 + const cells = [ + { id: 'reconcile-cell-a', url: 'https://reconcile-a.example.com', capacityRequests: 100 }, + { id: 'reconcile-cell-b', url: 'https://reconcile-b.example.com', capacityRequests: 100 } + ] + const identities = Array.from({ length: 12 }, (_, index) => ({ + userId: `reconcile-user-${index}`, + relayHostId: `reconcilehost${String(index).padStart(4, '0')}` + })) + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells(cells) + for (const identity of identities) await seedStore.assign(identity) + await database.query( + `UPDATE relay_assignments SET reserved_controls = 7, reserved_splices = 5 + WHERE user_id LIKE 'reconcile-user-%'` + ) + await database.query( + `UPDATE relay_cells SET reserved_requests = 42 + WHERE cell_id IN (?, ?)`, + ['reconcile-cell-a', 'reconcile-cell-b'] + ) + + const assignmentsLocked = signal() + const activityReachedAssignment = signal() + const continueReconciliation = signal() + let gateReconciliation = true + const reconcileDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if ( + gateReconciliation && + phase === 'after' && + sql.includes('SELECT assignment.* FROM relay_assignments assignment') + ) { + gateReconciliation = false + assignmentsLocked.resolve() + await continueReconciliation.promise + } + }) + const activityDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if (phase === 'before' && sql.includes('FROM relay_assignments WHERE user_id')) { + activityReachedAssignment.resolve() + } + }) + const reconcileStore = new RelayAssignmentStore(reconcileDatabase, () => now) + const activityStore = new RelayAssignmentStore(activityDatabase, () => now) + + const reconciliation = reconcileStore.cellEvacuationStatus( + 'reconcile-cell-a', + 'reconcile-cell-b', + true + ) + await assignmentsLocked.promise + const activity = activityStore.acquireActivity(identities[0]!, { + activityId: 'splice:reconcile', + kind: 'splice', + cellId: 'reconcile-cell-a' + }) + await activityReachedAssignment.promise + continueReconciliation.resolve() + await Promise.all([reconciliation, activity]) + + expect(reconcileDatabase.attempts).toBe(1) + expect(activityDatabase.attempts).toBe(1) + const reservations = await database.query( + `SELECT cell.cell_id, cell.reserved_requests, + COALESCE(SUM(lease.request_units), 0) AS lease_units + FROM relay_cells cell + LEFT JOIN relay_assignment_activity_leases lease ON lease.cell_id = cell.cell_id + WHERE cell.cell_id IN (?, ?) + GROUP BY cell.cell_id, cell.reserved_requests ORDER BY cell.cell_id`, + ['reconcile-cell-a', 'reconcile-cell-b'] + ) + expect(reservations).toEqual([ + { cell_id: 'reconcile-cell-a', reserved_requests: '8', lease_units: '8' }, + { cell_id: 'reconcile-cell-b', reserved_requests: '6', lease_units: '6' } + ]) + }, 15_000) + + it('fences a source activity queued behind evacuation completion', async () => { + const now = 1_400_000_000_000 + const identity = { + userId: 'completion-race-user', + relayHostId: 'completionrace01' + } + const sourceCellId = 'completion-race-source' + const targetCellId = 'completion-race-target' + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells([ + { + id: sourceCellId, + url: 'https://completion-source.example.com', + capacityRequests: 100 + }, + { + id: targetCellId, + url: 'https://completion-target.example.com', + capacityRequests: 100 + } + ]) + const assignment = await seedStore.assign(identity) + const sourceControl = await seedStore.activateControl(identity, { + cellId: sourceCellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + const migration = await seedStore.startEvacuation(identity, targetCellId) + await seedStore.activateControl(identity, { + cellId: targetCellId, + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await seedStore.markMigrationTargetRegistered(identity, { + cellId: targetCellId, + assignmentEpoch: migration.assignmentEpoch + }) + await seedStore.releaseActivity(identity, sourceControl) + + const assignmentLocked = signal() + const activityReachedAssignment = signal() + const continueCompletion = signal() + let gateCompletion = true + const completionDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if ( + gateCompletion && + phase === 'after' && + sql.includes('FROM relay_assignments WHERE user_id') + ) { + gateCompletion = false + assignmentLocked.resolve() + await continueCompletion.promise + } + }) + const activityDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if (phase === 'before' && sql.includes('FROM relay_assignments WHERE user_id')) { + activityReachedAssignment.resolve() + } + }) + const completionStore = new RelayAssignmentStore(completionDatabase, () => now) + const activityStore = new RelayAssignmentStore(activityDatabase, () => now) + + const completion = completionStore.completeReadyEvacuations() + await assignmentLocked.promise + const activity = activityStore.acquireActivity(identity, { + activityId: 'install:queued-source-work', + kind: 'install', + cellId: sourceCellId + }) + const activityRejected = expect(activity).rejects.toThrow( + 'activity_cell_not_authoritative' + ) + await activityReachedAssignment.promise + continueCompletion.resolve() + + await expect(completion).resolves.toBe(1) + await activityRejected + await expect( + seedStore.cellEvacuationStatus(sourceCellId, targetCellId, false) + ).resolves.toMatchObject({ inProgress: 0 }) + }, 15_000) + + it('defers completion instead of deadlocking with a legacy cell-first lock', async () => { + const now = 1_500_000_000_000 + const identity = { + userId: 'completion-contention-user', + relayHostId: 'completionwait01' + } + const sourceCell = { + id: 'completion-contention-source', + url: 'https://completion-contention-source.example.com', + capacityRequests: 100 + } + const targetCell = { + id: 'completion-contention-target', + url: 'https://completion-contention-target.example.com', + capacityRequests: 100 + } + const store = new RelayAssignmentStore(database, () => now, { + requireLiveCells: true + }) + await store.reconcileCells([sourceCell, targetCell]) + await store.recordCellHeartbeat({ + cellId: sourceCell.id, + cellUrl: sourceCell.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + startedAt: now - 100, + ready: true, + observedRequests: 0 + }) + await store.recordCellHeartbeat({ + cellId: targetCell.id, + cellUrl: targetCell.url, + cellIncarnation: '22222222-2222-4222-8222-222222222222', + startedAt: now - 100, + ready: true, + observedRequests: 0 + }) + const assignment = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: sourceCell.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + const migration = await store.startEvacuation(identity, targetCell.id) + await store.activateControl(identity, { + cellId: targetCell.id, + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: targetCell.id, + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + await store.setCellEnabled(sourceCell.id, false) + + const legacyCellLocked = signal() + const completionAssignmentLocked = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [ + sourceCell.id + ]) + legacyCellLocked.resolve() + await completionAssignmentLocked.promise + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + }) + await legacyCellLocked.promise + + let signalCompletion = true + const completionDatabase = new TransactionProbeDatabase( + database, + async (phase, sql) => { + if ( + signalCompletion && + phase === 'after' && + sql.includes('FROM relay_assignments WHERE user_id') + ) { + signalCompletion = false + completionAssignmentLocked.resolve() + } + } + ) + const completionStore = new RelayAssignmentStore(completionDatabase, () => now, { + requireLiveCells: true + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + await expect(completionStore.completeReadyEvacuations()).resolves.toBe(0) + await expect(legacyTransaction).resolves.toBeUndefined() + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('orca_relay_postgres_transaction_retry') + ) + await expect(store.completeReadyEvacuations()).resolves.toBe(1) + }, 15_000) + + it('normalizes persistent dead-source inventory contention', async () => { + let now = 1_550_000_000_000 + const identity = { + userId: 'dead-source-contention-user', + relayHostId: 'deadsourcewait01' + } + const secondIdentity = { + userId: identity.userId, + relayHostId: 'deadsourcewait02' + } + const sourceCell = { + id: 'dead-source-contention-source', + url: 'https://dead-source-contention-source.example.com', + capacityRequests: 100 + } + const targetCell = { + id: 'dead-source-contention-target', + url: 'https://dead-source-contention-target.example.com', + capacityRequests: 100 + } + const sourceIncarnation = '11111111-1111-4111-8111-111111111111' + const targetIncarnation = '22222222-2222-4222-8222-222222222222' + const store = new RelayAssignmentStore(database, () => now, { + requireLiveCells: true + }) + await store.reconcileCells([sourceCell, targetCell]) + await store.setCellEnabled(targetCell.id, false) + await store.recordCellHeartbeat({ + cellId: sourceCell.id, + cellUrl: sourceCell.url, + cellIncarnation: sourceIncarnation, + startedAt: now - 100, + ready: true, + observedRequests: 0 + }) + await store.recordCellHeartbeat({ + cellId: targetCell.id, + cellUrl: targetCell.url, + cellIncarnation: targetIncarnation, + startedAt: now - 100, + ready: true, + observedRequests: 0 + }) + const assignment = await store.assign(identity) + const sourceControl = await store.activateControl(identity, { + cellId: sourceCell.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.setCellEnabled(targetCell.id, true) + const migration = await store.startEvacuation(identity, targetCell.id) + await store.activateControl(identity, { + cellId: targetCell.id, + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: targetCell.id, + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, sourceControl) + const secondAssignment = await store.assign(secondIdentity) + expect(secondAssignment.cellId).toBe(sourceCell.id) + const secondSourceControl = await store.activateControl(secondIdentity, { + cellId: sourceCell.id, + assignmentEpoch: secondAssignment.assignmentEpoch, + generation: 1 + }) + const secondMigration = await store.startEvacuation(secondIdentity, targetCell.id) + await store.activateControl(secondIdentity, { + cellId: targetCell.id, + assignmentEpoch: secondMigration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(secondIdentity, { + cellId: targetCell.id, + assignmentEpoch: secondMigration.assignmentEpoch + }) + await store.releaseActivity(secondIdentity, secondSourceControl) + await store.setCellEnabled(sourceCell.id, false) + now += 45_001 + await store.recordCellHeartbeat({ + cellId: targetCell.id, + cellUrl: targetCell.url, + cellIncarnation: targetIncarnation, + startedAt: now - 45_101, + ready: true, + observedRequests: 0 + }) + await store.attestCellFence(sourceCell.id, sourceIncarnation) + + const legacyCellLocked = signal() + const completionAssignmentLocked = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [ + sourceCell.id + ]) + legacyCellLocked.resolve() + await completionAssignmentLocked.promise + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + }) + await legacyCellLocked.promise + + let signalCompletion = true + const fallbackInventoryLocked = signal() + const freshAssignmentLocked = signal() + const persistentInventoryLocked = signal() + let releasePersistentInventory = false + const completionDatabase = new TransactionProbeDatabase( + database, + async (phase, sql) => { + if ( + signalCompletion && + phase === 'after' && + sql.includes('FROM relay_assignments WHERE user_id') + ) { + signalCompletion = false + completionAssignmentLocked.resolve() + } else if ( + phase === 'after' && + sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC' + ) { + fallbackInventoryLocked.resolve() + await freshAssignmentLocked.promise + } + } + ) + const completionStore = new RelayAssignmentStore(completionDatabase, () => now, { + requireLiveCells: true + }) + const freshAssignmentTransaction = database.transaction(async (transaction) => { + await fallbackInventoryLocked.promise + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + freshAssignmentLocked.resolve() + await transaction.queryLocked(`SELECT * FROM relay_cells ORDER BY cell_id ASC`) + persistentInventoryLocked.resolve() + while (!releasePersistentInventory) { + await new Promise((resolve) => setTimeout(resolve, 250)) + await transaction.query(`SELECT 1`) + } + }) + + const completion = completionStore.cellEvacuationStatus( + sourceCell.id, + targetCell.id, + true + ) + await persistentInventoryLocked.promise + await expect( + completion + ).resolves.toMatchObject({ inProgress: 2, completed: 0, blocked: 2 }) + await expect(legacyTransaction).resolves.toBeUndefined() + releasePersistentInventory = true + await expect(freshAssignmentTransaction).resolves.toBeUndefined() + await expect( + store.cellEvacuationStatus(sourceCell.id, targetCell.id, true) + ).resolves.toMatchObject({ inProgress: 0, completed: 2, blocked: 0 }) + }, 25_000) + + it('defers expired lease cleanup around a legacy cell-first lock', async () => { + const now = 1_600_000_000_000 + const identity = { + userId: 'lease-contention-user', + relayHostId: 'leasecontention1' + } + const cell = { + id: 'lease-contention-cell', + url: 'https://lease-contention-cell.example.com', + capacityRequests: 100 + } + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells([cell]) + await store.assign(identity) + await database.query( + `UPDATE relay_assignment_activity_leases SET expires_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [now - 1, identity.userId, identity.relayHostId] + ) + + const legacyCellLocked = signal() + const cleanupAssignmentLocked = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cell.id]) + legacyCellLocked.resolve() + await cleanupAssignmentLocked.promise + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + }) + await legacyCellLocked.promise + + let signalCleanup = true + const cleanupDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if ( + signalCleanup && + phase === 'after' && + sql.includes('FROM relay_assignments WHERE user_id') + ) { + signalCleanup = false + cleanupAssignmentLocked.resolve() + } + }) + const cleanupStore = new RelayAssignmentStore(cleanupDatabase, () => now) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + await expect(cleanupStore.releaseExpiredActivityLeases()).resolves.toBe(0) + await expect(legacyTransaction).resolves.toBeUndefined() + expect(cleanupDatabase.attempts).toBe(1) + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('orca_relay_postgres_transaction_retry') + ) + await expect(store.releaseExpiredActivityLeases()).resolves.toBe(1) + }, 15_000) + + it('skips an expired lease when another director owns its assignment lock', async () => { + const now = 1_650_000_000_000 + const identity = { + userId: 'lease-assignment-contention-user', + relayHostId: 'leaseassignment1' + } + const cell = { + id: 'lease-assignment-contention-cell', + url: 'https://lease-assignment-contention-cell.example.com', + capacityRequests: 100 + } + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells([cell]) + await store.assign(identity) + await database.query( + `UPDATE relay_assignment_activity_leases SET expires_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [now - 1, identity.userId, identity.relayHostId] + ) + + const assignmentLocked = signal() + const releaseAssignment = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + assignmentLocked.resolve() + await releaseAssignment.promise + }) + await assignmentLocked.promise + + const cleanupDatabase = new TransactionProbeDatabase(database, async () => undefined) + const cleanupStore = new RelayAssignmentStore(cleanupDatabase, () => now) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + await expect(cleanupStore.releaseExpiredActivityLeases()).resolves.toBe(0) + expect(cleanupDatabase.attempts).toBe(1) + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('orca_relay_postgres_transaction_retry') + ) + warn.mockRestore() + + releaseAssignment.resolve() + await expect(legacyTransaction).resolves.toBeUndefined() + await expect(store.releaseExpiredActivityLeases()).resolves.toBe(1) + }, 15_000) + + it('defers aggregate expiry cleanup around a legacy cell-first lock', async () => { + const now = 1_700_000_000_000 + const identity = { + userId: 'aggregate-contention-user', + relayHostId: 'aggregatewait01' + } + const cell = { + id: 'aggregate-contention-cell', + url: 'https://aggregate-contention-cell.example.com', + capacityRequests: 100 + } + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells([cell]) + await store.assign(identity) + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database.query( + `UPDATE relay_assignments SET lease_expires_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [now - 1, identity.userId, identity.relayHostId] + ) + + const legacyCellLocked = signal() + const cleanupAssignmentsLocked = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cell.id]) + legacyCellLocked.resolve() + await cleanupAssignmentsLocked.promise + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + }) + await legacyCellLocked.promise + + let signalCleanup = true + const cleanupDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if ( + signalCleanup && + phase === 'after' && + sql.includes('FROM relay_assignments WHERE lease_expires_at') + ) { + signalCleanup = false + cleanupAssignmentsLocked.resolve() + } + }) + const cleanupStore = new RelayAssignmentStore(cleanupDatabase, () => now) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + await expect(cleanupStore.releaseExpiredActivity()).resolves.toBe(0) + await expect(legacyTransaction).resolves.toBeUndefined() + expect(cleanupDatabase.attempts).toBe(1) + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('orca_relay_postgres_transaction_retry') + ) + await expect(store.releaseExpiredActivity()).resolves.toBe(1) + }, 15_000) + + it('defers aggregate expiry before waiting on a legacy assignment row', async () => { + const now = 1_700_000_000_000 + const identity = { + userId: 'aggregate-contention-user', + relayHostId: 'aggregatewait01' + } + const cell = { + id: 'aggregate-contention-cell', + url: 'https://aggregate-contention-cell.example.com', + capacityRequests: 100 + } + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells([cell]) + await store.assign(identity) + await database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await database.query( + `UPDATE relay_assignments SET lease_expires_at = ? + WHERE user_id = ? AND relay_host_id = ?`, + [now - 1, identity.userId, identity.relayHostId] + ) + + const legacyAssignmentLocked = signal() + const releaseLegacyAssignment = signal() + const legacyTransaction = database.transaction(async (transaction) => { + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + legacyAssignmentLocked.resolve() + await releaseLegacyAssignment.promise + }) + await legacyAssignmentLocked.promise + + const timedOut = Symbol('timed-out') + const cleanup = store.releaseExpiredActivity() + const result = await Promise.race([ + cleanup, + new Promise((resolve) => { + setTimeout(() => resolve(timedOut), 750) + }) + ]) + releaseLegacyAssignment.resolve() + await legacyTransaction + if (result === timedOut) await cleanup + + expect(result).toBe(0) + await expect(store.releaseExpiredActivity()).resolves.toBe(1) + }, 15_000) + + it('keeps concurrent dormant reassignments capacity-consistent', async () => { + const now = 1_000_000_000_000 + const serializedDatabase = new TransactionProbeDatabase(database, async () => undefined) + const store = new RelayAssignmentStore(serializedDatabase, () => now) + await database.query( + `DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'postgres-user-%'` + ) + await database.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'postgres-user-%'`) + await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?, ?)`, [ + 'cell-a', + 'cell-b', + 'cell-c' + ]) + await store.reconcileCells([ + { id: 'cell-a', url: 'https://cell-a.example.com', capacityRequests: 100 }, + { id: 'cell-b', url: 'https://cell-b.example.com', capacityRequests: 100 }, + { id: 'cell-c', url: 'https://cell-c.example.com', capacityRequests: 100 } + ]) + const identities = Array.from({ length: 30 }, (_, index) => ({ + userId: `postgres-user-${index}`, + relayHostId: `postgreshost${String(index).padStart(5, '0')}` + })) + await Promise.all(identities.map(async (identity) => await store.assign(identity))) + + await database.query(`DELETE FROM relay_assignment_activity_leases`) + await database.query( + `UPDATE relay_assignments SET lease_expires_at = ?, last_activity_at = ?, + reserved_controls = 0, reserved_splices = 0, reserved_invites = 0, + pending_installs = 0, pending_confirmations = 0, migration_leases = 0`, + [0, 0] + ) + await database.query(`UPDATE relay_cells SET reserved_requests = 0`) + + await Promise.all(identities.map(async (identity) => await store.assign(identity))) + + const reservations = await database.query( + `SELECT cell_id, reserved_requests FROM relay_cells ORDER BY cell_id` + ) + expect(reservations).toEqual([ + { cell_id: 'cell-a', reserved_requests: '10' }, + { cell_id: 'cell-b', reserved_requests: '10' }, + { cell_id: 'cell-c', reserved_requests: '10' } + ]) + expect(serializedDatabase.attempts).toBe(121) + }, 10_000) + + it('renews independent sticky assignments concurrently without the placement queue', async () => { + const now = 1_800_000_000_000 + const identities = Array.from({ length: 6 }, (_, index) => ({ + userId: `sticky-fast-user-${index}`, + relayHostId: `stickyfast${String(index).padStart(6, '0')}` + })) + const cell = { + id: 'sticky-fast-cell', + url: 'https://sticky-fast.example.com', + capacityRequests: 100 + } + const seedStore = new RelayAssignmentStore(database, () => now) + await seedStore.reconcileCells([cell]) + for (const identity of identities) await seedStore.assign(identity) + + const allRenewalsReachedDatabase = signal() + const releaseRenewals = signal() + let renewalArrivals = 0 + const probedDatabase = new TransactionProbeDatabase(database, async (phase, sql) => { + if ( + phase !== 'after' || + !sql.includes('FROM relay_assignments WHERE user_id = ?') + ) { + return + } + renewalArrivals++ + if (renewalArrivals === identities.length) allRenewalsReachedDatabase.resolve() + await releaseRenewals.promise + }) + const store = new RelayAssignmentStore(probedDatabase, () => now) + const renewals = identities.map(async (identity) => await store.assign(identity)) + let reachedBeforeTimeout = false + try { + reachedBeforeTimeout = await Promise.race([ + allRenewalsReachedDatabase.promise.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 1_000)) + ]) + } finally { + releaseRenewals.resolve() + await Promise.all(renewals) + } + + expect(reachedBeforeTimeout).toBe(true) + expect(renewalArrivals).toBe(identities.length) + expect(probedDatabase.attempts).toBe(identities.length) + const state = await database.query( + `SELECT COUNT(*) AS assignments, SUM(reserved_controls) AS controls + FROM relay_assignments WHERE user_id LIKE 'sticky-fast-user-%'` + ) + expect(state).toEqual([{ assignments: '6', controls: '6' }]) + }, 10_000) +}) diff --git a/cloud/apps/relay/src/public-assignment-admission.test.ts b/cloud/apps/relay/src/public-assignment-admission.test.ts new file mode 100644 index 00000000000..dbc9a7278be --- /dev/null +++ b/cloud/apps/relay/src/public-assignment-admission.test.ts @@ -0,0 +1,334 @@ +import { describe, expect, it, vi } from 'vitest' +import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js' + +describe('public assignment admission', () => { + it('bounds global concurrency and repeated work for one relay host', async () => { + let now = 0 + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 2, + minIntervalMs: 5_000, + now: () => now + }) + const first = await admission.acquire('host-a') + expect(first).not.toBeNull() + await expect(admission.acquire('host-a')).resolves.toBeNull() + const second = await admission.acquire('host-b') + expect(second).not.toBeNull() + await expect(admission.acquire('host-c')).resolves.toBeNull() + + first?.release() + await expect(admission.acquire('host-a')).resolves.toBeNull() + now = 5_000 + const recovered = await admission.acquire('host-a') + expect(recovered).not.toBeNull() + recovered?.release() + second?.release() + }) + + it('releases capacity once when callers settle more than once', async () => { + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + minIntervalMs: 0, + now: vi.fn(() => 0) + }) + const lease = await admission.acquire('host-a') + lease?.release() + lease?.release() + await expect(admission.acquire('host-b')).resolves.not.toBeNull() + }) + + it('grants ordinary waiters in FIFO order without raising concurrency', async () => { + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + maxQueued: 2, + waitMs: 1_000, + minIntervalMs: 0, + schedule: () => () => undefined + }) + const first = await admission.acquire('host-a') + const secondPromise = admission.acquire('host-b') + const thirdPromise = admission.acquire('host-c') + + await expect(admission.acquire('host-d')).resolves.toBeNull() + first?.release() + const second = await secondPromise + expect(second).not.toBeNull() + let thirdSettled = false + void thirdPromise.then(() => { thirdSettled = true }) + await Promise.resolve() + expect(thirdSettled).toBe(false) + second?.release() + const third = await thirdPromise + expect(third).not.toBeNull() + third?.release() + }) + + it('fails an ordinary wait closed when its bounded wait expires', async () => { + let expire!: () => void + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + maxQueued: 1, + waitMs: 1_000, + minIntervalMs: 0, + schedule: (callback) => { + expire = callback + return () => undefined + } + }) + const first = await admission.acquire('host-a') + const waiting = admission.acquire('host-b') + + expire() + await expect(waiting).resolves.toBeNull() + first?.release() + }) + + it('gives a bounded reserved waiter the next public slot', async () => { + let expire!: () => void + let cancelled = false + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 2, + maxReservedConcurrent: 1, + reservedWaitMs: 1_000, + minIntervalMs: 0, + schedule: (callback) => { + expire = callback + return () => { + cancelled = true + } + } + }) + const first = await admission.acquire('host-a') + const second = await admission.acquire('host-b') + const reservedPromise = admission.acquireReserved('host-c') + + await expect(admission.acquire('host-d')).resolves.toBeNull() + await expect(admission.acquireReserved('host-e')).resolves.toBeNull() + first?.release() + const reserved = await reservedPromise + + expect(reserved).not.toBeNull() + expect(cancelled).toBe(true) + await expect(admission.acquire('host-d')).resolves.toBeNull() + reserved?.release() + await expect(admission.acquire('host-d')).resolves.not.toBeNull() + second?.release() + expire() + }) + + it('lets reserved recovery displace the same host from the ordinary queue', async () => { + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + maxQueued: 1, + waitMs: 1_000, + maxReservedConcurrent: 1, + reservedWaitMs: 1_000, + minIntervalMs: 0, + schedule: () => () => undefined + }) + const active = await admission.acquire('host-a') + const ordinary = admission.acquire('host-b') + const reserved = admission.acquireReserved('host-b') + + await expect(ordinary).resolves.toBeNull() + active?.release() + const recovered = await reserved + expect(recovered).not.toBeNull() + recovered?.release() + }) + + it('fails a reserved wait closed when its bounded wait expires', async () => { + let expire!: () => void + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 2, + maxReservedConcurrent: 1, + reservedWaitMs: 1_000, + minIntervalMs: 0, + schedule: (callback) => { + expire = callback + return () => undefined + } + }) + const first = await admission.acquire('host-a') + const second = await admission.acquire('host-b') + const reserved = admission.acquireReserved('host-c') + + expire() + await expect(reserved).resolves.toBeNull() + first?.release() + await expect(admission.acquire('host-d')).resolves.not.toBeNull() + second?.release() + }) + + it('serializes same-host assignment and reserved recovery without losing priority', async () => { + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 2, + maxReservedConcurrent: 1, + reservedWaitMs: 1_000, + minIntervalMs: 0, + schedule: () => () => undefined + }) + const assignment = await admission.acquire('host-a') + const reservedPromise = admission.acquireReserved('host-a') + + await Promise.resolve() + await expect(admission.acquire('host-b')).resolves.toBeNull() + assignment?.release() + const reserved = await reservedPromise + + expect(reserved).not.toBeNull() + await expect(admission.acquire('host-a')).resolves.toBeNull() + reserved?.release() + await expect(admission.acquire('host-b')).resolves.not.toBeNull() + }) + + it('separates a self-throttled host from a saturated director', async () => { + let now = 0 + const reasons: string[] = [] + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + maxQueued: 0, + minIntervalMs: 5_000, + now: () => now, + onRejected: (reason) => reasons.push(reason) + }) + const held = await admission.acquire('host-a') + + // Same host while its own attempt is still running. + await expect(admission.acquire('host-a')).resolves.toBeNull() + // A different host with the single slot taken and no queue. + await expect(admission.acquire('host-b')).resolves.toBeNull() + held?.release() + // Same host again, now inside its retry penalty rather than in flight. + await expect(admission.acquire('host-a')).resolves.toBeNull() + + expect(reasons).toEqual(['host-in-flight', 'queue-full', 'host-rate-limited']) + + now = 5_000 + const recovered = await admission.acquire('host-a') + expect(recovered).not.toBeNull() + expect(reasons).toHaveLength(3) + recovered?.release() + }) + + it('reports a timed-out wait separately from a full queue', async () => { + let expire!: () => void + const reasons: string[] = [] + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + maxQueued: 1, + waitMs: 1_000, + minIntervalMs: 0, + schedule: (callback) => { + expire = callback + return () => undefined + }, + onRejected: (reason) => reasons.push(reason) + }) + const first = await admission.acquire('host-a') + const waiting = admission.acquire('host-b') + + expire() + await expect(waiting).resolves.toBeNull() + expect(reasons).toEqual(['wait-timeout']) + first?.release() + }) + + it('reports a displaced queue entry as superseded, not as a timeout', async () => { + const reasons: string[] = [] + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + maxQueued: 1, + waitMs: 1_000, + maxReservedConcurrent: 1, + reservedWaitMs: 1_000, + minIntervalMs: 0, + schedule: () => () => undefined, + onRejected: (reason) => reasons.push(reason) + }) + const active = await admission.acquire('host-a') + const ordinary = admission.acquire('host-b') + const reserved = admission.acquireReserved('host-b') + + await expect(ordinary).resolves.toBeNull() + expect(reasons).toEqual(['superseded']) + active?.release() + const recovered = await reserved + expect(recovered).not.toBeNull() + recovered?.release() + }) + + it('distinguishes a busy reserved lane from a reserved host already in flight', async () => { + const reasons: string[] = [] + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 4, + // Two slots so the lane still has room when the same host asks twice. + maxReservedConcurrent: 2, + reservedWaitMs: 1_000, + minIntervalMs: 0, + schedule: () => () => undefined, + onRejected: (reason) => reasons.push(reason) + }) + const reserved = await admission.acquireReserved('host-a') + expect(reserved).not.toBeNull() + + await expect(admission.acquireReserved('host-a')).resolves.toBeNull() + expect(reasons).toEqual(['host-in-flight']) + + const second = await admission.acquireReserved('host-b') + expect(second).not.toBeNull() + await expect(admission.acquireReserved('host-c')).resolves.toBeNull() + + expect(reasons).toEqual(['host-in-flight', 'reserved-unavailable']) + reserved?.release() + second?.release() + }) + + it('tells each caller its own rejection reason without crossing requests', async () => { + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 1, + maxQueued: 1, + waitMs: 1_000, + maxReservedConcurrent: 1, + reservedWaitMs: 1_000, + minIntervalMs: 5_000, + now: () => 0, + schedule: () => () => undefined + }) + const queued: string[] = [] + const displaced: string[] = [] + const throttled: string[] = [] + + const active = await admission.acquire('host-a') + // 'host-b' waits, then reserved recovery for the same host displaces it. + const ordinary = admission.acquire('host-b', (reason) => displaced.push(reason)) + const reserved = admission.acquireReserved('host-b', (reason) => queued.push(reason)) + await expect(ordinary).resolves.toBeNull() + active?.release() + const recovered = await reserved + recovered?.release() + await admission.acquire('host-a', (reason) => throttled.push(reason)) + + expect(displaced).toEqual(['superseded']) + expect(queued).toEqual([]) + expect(throttled).toEqual(['host-rate-limited']) + }) + + it('stays silent on every granted acquisition', async () => { + const reasons: string[] = [] + const admission = new RelayPublicAssignmentAdmission({ + maxConcurrent: 2, + maxReservedConcurrent: 1, + minIntervalMs: 0, + onRejected: (reason) => reasons.push(reason) + }) + const first = await admission.acquire('host-a') + const second = await admission.acquireReserved('host-b') + + expect(first).not.toBeNull() + expect(second).not.toBeNull() + expect(reasons).toEqual([]) + first?.release() + second?.release() + }) +}) diff --git a/cloud/apps/relay/src/public-assignment-admission.ts b/cloud/apps/relay/src/public-assignment-admission.ts new file mode 100644 index 00000000000..bd6d7f1a488 --- /dev/null +++ b/cloud/apps/relay/src/public-assignment-admission.ts @@ -0,0 +1,239 @@ +type AssignmentAdmissionLease = { release(): void } +type CancelWait = () => void +// Per-call sink: acquire() keeps returning null so callers stay unchanged, and the +// reason rides out of band to whoever made this particular request. +type RejectionSink = (reason: AssignmentAdmissionRejection) => void +type PendingAssignment = { + relayHostId: string + resolve: (lease: AssignmentAdmissionLease | null) => void + cancelWait: CancelWait + notifyRejected?: RejectionSink +} + +// Every rejection here becomes an identical 503, so without the reason a busy +// director and a self-throttling host are indistinguishable in production. +export type AssignmentAdmissionRejection = + | 'host-in-flight' + | 'host-rate-limited' + | 'queue-full' + | 'wait-timeout' + | 'superseded' + | 'reserved-unavailable' + +const MAX_TRACKED_HOSTS = 4_096 + +export class RelayPublicAssignmentAdmission { + private active = 0 + private activeReserved = 0 + private readonly activeAssignmentHosts = new Set() + private readonly activeReservedHosts = new Set() + private readonly queuedAssignmentHosts = new Set() + private readonly lastAttemptByHost = new Map() + private readonly lastReservedAttemptByHost = new Map() + private readonly pendingAssignments: PendingAssignment[] = [] + private pendingReserved: PendingAssignment | undefined + + constructor( + private readonly options: { + maxConcurrent: number + maxQueued?: number + waitMs?: number + maxReservedConcurrent?: number + reservedWaitMs?: number + minIntervalMs: number + now?: () => number + schedule?: (callback: () => void, delayMs: number) => CancelWait + onRejected?: (reason: AssignmentAdmissionRejection) => void + } + ) {} + + async acquire( + relayHostId: string, + notifyRejected?: RejectionSink + ): Promise { + const now = (this.options.now ?? Date.now)() + const lastAttempt = this.lastAttemptByHost.get(relayHostId) + if ( + this.activeAssignmentHosts.has(relayHostId) || + this.activeReservedHosts.has(relayHostId) || + this.queuedAssignmentHosts.has(relayHostId) || + this.pendingReserved?.relayHostId === relayHostId + ) { + return this.reject('host-in-flight', notifyRejected) + } + if (lastAttempt !== undefined && now - lastAttempt < this.options.minIntervalMs) { + return this.reject('host-rate-limited', notifyRejected) + } + if ( + this.active < this.options.maxConcurrent && + this.pendingReserved === undefined && + this.pendingAssignments.length === 0 + ) { + this.recordAttempt(this.lastAttemptByHost, relayHostId, now) + return this.createLease(relayHostId, false) + } + if (this.pendingAssignments.length >= (this.options.maxQueued ?? 0)) { + return this.reject('queue-full', notifyRejected) + } + + return await new Promise((resolve) => { + const schedule = this.options.schedule ?? defaultSchedule + let cancelWait: CancelWait = () => undefined + const pending: PendingAssignment = { + relayHostId, + resolve, + cancelWait: () => cancelWait(), + notifyRejected + } + this.pendingAssignments.push(pending) + this.queuedAssignmentHosts.add(relayHostId) + cancelWait = schedule(() => { + const index = this.pendingAssignments.indexOf(pending) + if (index === -1) return + this.pendingAssignments.splice(index, 1) + this.queuedAssignmentHosts.delete(relayHostId) + resolve(this.reject('wait-timeout', notifyRejected)) + }, this.options.waitMs ?? 1_000) + }) + } + + async acquireReserved( + relayHostId: string, + notifyRejected?: RejectionSink + ): Promise { + const maxReservedConcurrent = this.options.maxReservedConcurrent ?? 0 + const now = (this.options.now ?? Date.now)() + const lastAttempt = this.lastReservedAttemptByHost.get(relayHostId) + if (maxReservedConcurrent === 0 || this.activeReserved >= maxReservedConcurrent) { + return this.reject('reserved-unavailable', notifyRejected) + } + if (this.activeReservedHosts.has(relayHostId)) { + return this.reject('host-in-flight', notifyRejected) + } + if (this.pendingReserved !== undefined) { + return this.reject('reserved-unavailable', notifyRejected) + } + if (lastAttempt !== undefined && now - lastAttempt < this.options.minIntervalMs) { + return this.reject('host-rate-limited', notifyRejected) + } + this.cancelQueuedAssignment(relayHostId) + if ( + this.active < this.options.maxConcurrent && + !this.activeAssignmentHosts.has(relayHostId) + ) { + this.recordAttempt(this.lastReservedAttemptByHost, relayHostId, now) + return this.createLease(relayHostId, true) + } + + return await new Promise((resolve) => { + const schedule = this.options.schedule ?? defaultSchedule + let cancelWait: CancelWait = () => undefined + this.pendingReserved = { + relayHostId, + resolve, + cancelWait: () => cancelWait(), + notifyRejected + } + cancelWait = schedule(() => { + if (this.pendingReserved?.relayHostId !== relayHostId) return + this.pendingReserved = undefined + resolve(this.reject('wait-timeout', notifyRejected)) + this.grantPendingAssignments() + }, this.options.reservedWaitMs ?? 1_000) + }) + } + + private createLease(relayHostId: string, reserved: boolean): AssignmentAdmissionLease { + this.active++ + if (reserved) { + this.activeReserved++ + this.activeReservedHosts.add(relayHostId) + } else { + this.activeAssignmentHosts.add(relayHostId) + } + let released = false + return { + release: () => { + if (released) return + released = true + this.active = Math.max(0, this.active - 1) + if (reserved) { + this.activeReserved = Math.max(0, this.activeReserved - 1) + this.activeReservedHosts.delete(relayHostId) + } else { + this.activeAssignmentHosts.delete(relayHostId) + } + this.grantPendingReserved() + this.grantPendingAssignments() + } + } + } + + private grantPendingReserved(): void { + const pending = this.pendingReserved + if ( + !pending || + this.active >= this.options.maxConcurrent || + this.activeAssignmentHosts.has(pending.relayHostId) + ) { + return + } + this.pendingReserved = undefined + pending.cancelWait() + this.recordAttempt( + this.lastReservedAttemptByHost, + pending.relayHostId, + (this.options.now ?? Date.now)() + ) + pending.resolve(this.createLease(pending.relayHostId, true)) + } + + private grantPendingAssignments(): void { + while ( + this.pendingReserved === undefined && + this.active < this.options.maxConcurrent && + this.pendingAssignments.length > 0 + ) { + const pending = this.pendingAssignments.shift()! + this.queuedAssignmentHosts.delete(pending.relayHostId) + pending.cancelWait() + this.recordAttempt( + this.lastAttemptByHost, + pending.relayHostId, + (this.options.now ?? Date.now)() + ) + pending.resolve(this.createLease(pending.relayHostId, false)) + } + } + + private cancelQueuedAssignment(relayHostId: string): void { + const index = this.pendingAssignments.findIndex( + (pending) => pending.relayHostId === relayHostId + ) + if (index === -1) return + const [pending] = this.pendingAssignments.splice(index, 1) + this.queuedAssignmentHosts.delete(relayHostId) + pending?.cancelWait() + // The sink rides on the pending record: this rejects a different caller's request. + pending?.resolve(this.reject('superseded', pending.notifyRejected)) + } + + private reject(reason: AssignmentAdmissionRejection, notifyRejected?: RejectionSink): null { + this.options.onRejected?.(reason) + notifyRejected?.(reason) + return null + } + + private recordAttempt(attempts: Map, relayHostId: string, now: number): void { + attempts.delete(relayHostId) + attempts.set(relayHostId, now) + if (attempts.size > MAX_TRACKED_HOSTS) { + attempts.delete(attempts.keys().next().value!) + } + } +} + +function defaultSchedule(callback: () => void, delayMs: number): CancelWait { + const timer = setTimeout(callback, delayMs) + return () => clearTimeout(timer) +} diff --git a/cloud/apps/relay/src/public-assignment-circuit-breaker.test.ts b/cloud/apps/relay/src/public-assignment-circuit-breaker.test.ts new file mode 100644 index 00000000000..94be0536497 --- /dev/null +++ b/cloud/apps/relay/src/public-assignment-circuit-breaker.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it, vi } from 'vitest' +import { createRelayApp } from './app.js' +import type { RelayConfig } from './config.js' + +describe('public assignment circuit breaker', () => { + it('rejects assign and resolve without invoking relay state', async () => { + const assign = vi.fn() + const resolveResume = vi.fn() + const app = createRelayApp(config(), { + store: { resolveResume } as never, + assignments: { assign } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + for (const path of ['/v1/assign', '/v1/resolve']) { + const response = await app.request(path, { method: 'POST' }) + expect(response.status).toBe(503) + expect(response.headers.get('retry-after')).toBe('5') + expect(await response.json()).toEqual({ error: 'assignments_temporarily_unavailable' }) + } + expect(assign).not.toHaveBeenCalled() + expect(resolveResume).not.toHaveBeenCalled() + expect((await app.request('/health')).status).toBe(200) + expect((await app.request('/v1/admin/drain', { method: 'POST' })).status).toBe(401) + }) +}) + +function config(): RelayConfig { + return { + port: 8080, + publicUrl: 'https://relay.example.test', + cellUrl: 'https://relay.example.test', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'), + role: 'director', + cellId: 'director', + cells: [], + adminAudience: 'https://relay.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + runtimeServiceAccount: 'runtime@example.test', + adminJwksUrl: 'https://auth.example.test/jwks', + databasePoolMax: 3, + publicAssignmentsEnabled: false, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './data' + } +} diff --git a/cloud/apps/relay/src/public-assignment-overload.test.ts b/cloud/apps/relay/src/public-assignment-overload.test.ts new file mode 100644 index 00000000000..2d0426893c4 --- /dev/null +++ b/cloud/apps/relay/src/public-assignment-overload.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RelayAssignment } from './assignment-store.js' +import type { RelayConfig } from './config.js' + +const fakes = vi.hoisted(() => ({ + verifyRelayToken: vi.fn(async (token: string) => ({ + sub: 'user-1', + relayHostId: token + })) +})) + +vi.mock('./relay-token-verifier.js', () => ({ + createRelayTokenVerifier: () => fakes.verifyRelayToken, + readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null +})) + +import { createRelayApp } from './app.js' + +describe('public assignment overload', () => { + it('rejects duplicate and excess work before it reaches assignment state', async () => { + const hostA = 'aaaaaaaaaaaaaaaa' + const hostB = 'bbbbbbbbbbbbbbbb' + const hostC = 'cccccccccccccccc' + const pending = new Map>>() + const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => { + const operation = deferred() + pending.set(relayHostId, operation) + return await operation.promise + }) + const app = createRelayApp(config({ publicAssignmentQueueMax: 0 }), { + store: {} as never, + assignments: { assign } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const first = app.request('/v1/assign', assignmentRequest(hostA)) + await vi.waitFor(() => expect(pending.has(hostA)).toBe(true)) + const duplicate = await app.request('/v1/assign', assignmentRequest(hostA)) + expect(duplicate.status).toBe(503) + + const second = app.request('/v1/assign', assignmentRequest(hostB)) + await vi.waitFor(() => expect(pending.has(hostB)).toBe(true)) + const excess = await app.request('/v1/assign', assignmentRequest(hostC)) + expect(excess.status).toBe(503) + expect(excess.headers.get('retry-after')).toBe('5') + expect(assign).toHaveBeenCalledTimes(2) + + pending.get(hostA)?.resolve(assignment('cell-a', hostA)) + pending.get(hostB)?.resolve(assignment('cell-b', hostB)) + expect((await first).status).toBe(200) + expect((await second).status).toBe(200) + expect((await app.request('/v1/assign', assignmentRequest(hostA))).status).toBe(503) + expect(assign).toHaveBeenCalledTimes(2) + }) + + it('fairly drains a bounded incident-scale burst without raising database concurrency', async () => { + const pending = new Map>>() + const admittedHosts: string[] = [] + let active = 0 + let highWater = 0 + const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => { + admittedHosts.push(relayHostId) + active++ + highWater = Math.max(highWater, active) + const operation = deferred() + pending.set(relayHostId, operation) + try { + return await operation.promise + } finally { + active-- + pending.delete(relayHostId) + } + }) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + const hosts = Array.from( + { length: 430 }, + (_, index) => `host-${String(index).padStart(11, '0')}` + ) + + const firstWave = hosts.map((host) => + app.request('/v1/assign', assignmentRequest(host)) + ) + await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2)) + const retryWave = await Promise.all( + hosts.map((host) => app.request('/v1/assign', assignmentRequest(host))) + ) + + expect(retryWave.every((response) => response.status === 503)).toBe(true) + expect(assign).toHaveBeenCalledTimes(2) + while (assign.mock.calls.length < 130) { + const activeOperations = [...pending] + const expectedCalls = assign.mock.calls.length + activeOperations.length + for (const [host, operation] of activeOperations) { + operation.resolve(assignment(`cell-${host}`, host)) + } + await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(expectedCalls)) + } + for (const [host, operation] of pending) { + operation.resolve(assignment(`cell-${host}`, host)) + } + const firstResponses = await Promise.all(firstWave) + expect(firstResponses.filter((response) => response.status === 200)).toHaveLength(130) + expect(firstResponses.filter((response) => response.status === 503)).toHaveLength(300) + expect(admittedHosts).toEqual(hosts.slice(0, 130)) + expect(highWater).toBe(2) + }) + + it('reserves one bounded resolve lane while assignment work is saturated', async () => { + const pendingAssignments = new Map>>() + const pendingResolve = deferred<{ userId: string; relayDeviceId: string } | null>() + const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => { + const operation = deferred() + pendingAssignments.set(relayHostId, operation) + return await operation.promise + }) + const resolveResume = vi.fn(async () => await pendingResolve.promise) + const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) => + assignment('target-cell', relayHostId) + ) + const app = createRelayApp(config({ publicAssignmentQueueMax: 0 }), { + store: { resolveResume } as never, + assignments: { assign, resolve } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const assignmentA = app.request('/v1/assign', assignmentRequest('aaaaaaaaaaaaaaaa')) + const assignmentB = app.request('/v1/assign', assignmentRequest('bbbbbbbbbbbbbbbb')) + await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2)) + + const recovery = app.request('/v1/resolve', resolveRequest('cccccccccccccccc', 1)) + await Promise.resolve() + expect(resolveResume).not.toHaveBeenCalled() + const excessResolve = await app.request( + '/v1/resolve', + resolveRequest('dddddddddddddddd', 2) + ) + const excessAssign = await app.request('/v1/assign', assignmentRequest('eeeeeeeeeeeeeeee')) + + expect(excessResolve.status).toBe(503) + expect(excessAssign.status).toBe(503) + expect(assign).toHaveBeenCalledTimes(2) + expect(resolveResume).not.toHaveBeenCalled() + + pendingAssignments + .get('aaaaaaaaaaaaaaaa') + ?.resolve(assignment('cell-a', 'aaaaaaaaaaaaaaaa')) + expect((await assignmentA).status).toBe(200) + await vi.waitFor(() => expect(resolveResume).toHaveBeenCalledTimes(1)) + pendingResolve.resolve({ userId: 'user-1', relayDeviceId: 'device-1' }) + expect((await recovery).status).toBe(200) + expect(resolve).toHaveBeenCalledTimes(1) + + for (const [host, operation] of pendingAssignments) { + operation.resolve(assignment(`cell-${host}`, host)) + } + expect((await assignmentB).status).toBe(200) + }) +}) + +function assignmentRequest(relayHostId: string): RequestInit { + return { + method: 'POST', + headers: { + authorization: `Bearer ${relayHostId}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, relayHostId }) + } +} + +function resolveRequest(relayHostId: string, fill: number): RequestInit { + return { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + relayHostId, + resumeToken: Buffer.alloc(32, fill).toString('base64url') + }) + } +} + +function assignment(cellId: string, relayHostId: string): RelayAssignment { + return { + userId: 'user-1', + relayHostId, + cellId, + cellUrl: `https://${cellId}.relay.example.test`, + assignmentEpoch: 1, + leaseExpiresAt: Date.now() + 300_000 + } +} + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +function config(overrides: Partial = {}): RelayConfig { + return { + port: 8080, + publicUrl: 'https://relay.example.test', + cellUrl: 'https://relay.example.test', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'), + role: 'director', + cellId: 'director', + cells: [], + adminAudience: 'https://relay.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + runtimeServiceAccount: 'runtime@example.test', + adminJwksUrl: 'https://auth.example.test/jwks', + databasePoolMax: 3, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './data', + ...overrides + } +} diff --git a/cloud/apps/relay/src/public-assignment-reconnect-lane.test.ts b/cloud/apps/relay/src/public-assignment-reconnect-lane.test.ts new file mode 100644 index 00000000000..145d2c737cb --- /dev/null +++ b/cloud/apps/relay/src/public-assignment-reconnect-lane.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RelayAssignment } from './assignment-store.js' +import type { RelayConfig } from './config.js' + +const fakes = vi.hoisted(() => ({ + verifyRelayToken: vi.fn(async (token: string) => ({ + sub: 'user-1', + relayHostId: token + })) +})) + +vi.mock('./relay-token-verifier.js', () => ({ + createRelayTokenVerifier: () => fakes.verifyRelayToken, + readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null +})) + +import { createRelayApp } from './app.js' + +describe('public assignment reconnect lane', () => { + it('admits a verified reconnect past a saturated placement queue', async () => { + const reconnecting = 'rrrrrrrrrrrrrrrr' + const newcomerA = 'aaaaaaaaaaaaaaaa' + const newcomerB = 'bbbbbbbbbbbbbbbb' + const blocked = 'cccccccccccccccc' + const pending = new Map>>() + const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => { + if (relayHostId === reconnecting) return assignment('cell-r', reconnecting) + const operation = deferred() + pending.set(relayHostId, operation) + return await operation.promise + }) + const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) => + relayHostId === reconnecting ? assignment('cell-r', reconnecting) : null + ) + const outcomes: string[] = [] + const app = createRelayApp(config({ publicAssignmentQueueMax: 0 }), { + store: {} as never, + assignments: { assign, resolve } as never, + drain: vi.fn(), + ready: vi.fn(async () => true), + recordAssignmentAdmission: (outcome) => outcomes.push(outcome) + }) + + // Saturate the placement lane with two in-flight newcomers, zero queue. + const first = app.request('/v1/assign', assignmentRequest(newcomerA)) + await vi.waitFor(() => expect(pending.has(newcomerA)).toBe(true)) + const second = app.request('/v1/assign', assignmentRequest(newcomerB)) + await vi.waitFor(() => expect(pending.has(newcomerB)).toBe(true)) + expect((await app.request('/v1/assign', assignmentRequest(blocked))).status).toBe(503) + + const reconnected = await app.request( + '/v1/assign', + assignmentRequest(reconnecting, { reconnect: true }) + ) + + expect(reconnected.status).toBe(200) + expect(resolve).toHaveBeenCalledWith({ userId: 'user-1', relayHostId: reconnecting }) + expect(assign).toHaveBeenCalledWith({ userId: 'user-1', relayHostId: reconnecting }) + expect(outcomes).toContain('sticky') + + pending.get(newcomerA)?.resolve(assignment('cell-a', newcomerA)) + pending.get(newcomerB)?.resolve(assignment('cell-b', newcomerB)) + expect((await first).status).toBe(200) + expect((await second).status).toBe(200) + }) + + it('rate-limits repeat fast-lane attempts with the short retry-after', async () => { + const host = 'rrrrrrrrrrrrrrrr' + const assign = vi.fn(async () => assignment('cell-r', host)) + const resolve = vi.fn(async () => assignment('cell-r', host)) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, resolve } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + expect( + (await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))).status + ).toBe(200) + const repeat = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true })) + + expect(repeat.status).toBe(503) + expect(repeat.headers.get('retry-after')).toBe('2') + expect(resolve).toHaveBeenCalledTimes(1) + expect(assign).toHaveBeenCalledTimes(1) + }) + + it('sends an unverified reconnect hint through the placement lane unchanged', async () => { + const host = 'nnnnnnnnnnnnnnnn' + const assign = vi.fn(async () => assignment('cell-n', host)) + const resolve = vi.fn(async () => null) + const outcomes: string[] = [] + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, resolve } as never, + drain: vi.fn(), + ready: vi.fn(async () => true), + recordAssignmentAdmission: (outcome) => outcomes.push(outcome) + }) + + const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true })) + + expect(response.status).toBe(200) + expect(assign).toHaveBeenCalledTimes(1) + expect(outcomes).toEqual(['placement']) + }) + + it('rejects on the fast lane when the verification probe fails transiently', async () => { + const host = 'tttttttttttttttt' + const assign = vi.fn() + const resolve = vi.fn(async () => { + throw Object.assign(new Error('deadlock detected'), { code: '40P01' }) + }) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, resolve } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true })) + + expect(response.status).toBe(503) + expect(response.headers.get('retry-after')).toBe('2') + expect(assign).not.toHaveBeenCalled() + }) + + it('never probes for unhinted requests', async () => { + const host = 'uuuuuuuuuuuuuuuu' + const assign = vi.fn(async () => assignment('cell-u', host)) + const resolve = vi.fn() + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, resolve } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200) + expect(resolve).not.toHaveBeenCalled() + }) +}) + +function assignmentRequest(relayHostId: string, extra: { reconnect?: boolean } = {}): RequestInit { + return { + method: 'POST', + headers: { + authorization: `Bearer ${relayHostId}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, relayHostId, ...extra }) + } +} + +function assignment(cellId: string, relayHostId: string): RelayAssignment { + return { + userId: 'user-1', + relayHostId, + cellId, + cellUrl: `https://${cellId}.relay.example.test`, + assignmentEpoch: 1, + leaseExpiresAt: Date.now() + 300_000 + } +} + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +function config(overrides: Partial = {}): RelayConfig { + return { + port: 8080, + publicUrl: 'https://relay.example.test', + cellUrl: 'https://relay.example.test', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'), + role: 'director', + cellId: 'director', + cells: [], + adminAudience: 'https://relay.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + runtimeServiceAccount: 'runtime@example.test', + adminJwksUrl: 'https://auth.example.test/jwks', + databasePoolMax: 3, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './data', + ...overrides + } +} diff --git a/cloud/apps/relay/src/regional-host-drain-app.test.ts b/cloud/apps/relay/src/regional-host-drain-app.test.ts new file mode 100644 index 00000000000..1cd34902520 --- /dev/null +++ b/cloud/apps/relay/src/regional-host-drain-app.test.ts @@ -0,0 +1,536 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RelayConfig } from './config.js' + +vi.mock('./admin-token-verifier.js', () => ({ + createAdminTokenVerifier: () => async (token: string, route?: string) => + token === 'deploy-token' || + (token === 'monitor-token' && + (!route || route === '/v1/admin/regional-rehome-control')), + createReadOnlyAdminTokenVerifier: () => async () => false, + createRegionalRehomeControlApplyTokenVerifier: () => async (token: string) => + token === 'deploy-token', + createRegionalRehomeRuntimeTokenVerifier: () => async (token: string) => + token === 'runtime-token', + createRegionalRehomeTokenVerifier: () => async (token: string) => token === 'rehome-token', + createRuntimeTokenVerifier: () => async (token: string) => token === 'runtime-token' +})) + +vi.mock('./relay-token-verifier.js', () => ({ + createRelayTokenVerifier: () => async () => null, + readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null +})) + +import { createRelayApp } from './app.js' +import { RelayObservability } from './relay-observability.js' +import { emptyPostgresPoolPressureCounts } from './postgres-pool-pressure.js' +import { + REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + REGIONAL_REHOME_TRUST_PROBE_USER_ID +} from './regional-rehome-trust-probe.js' + +const cellIncarnation = '11111111-1111-4111-8111-111111111111' +const request = { + v: 1, + attemptId: '22222222-2222-4222-8222-222222222222', + userId: 'user-1', + relayHostId: 'abcdefghijklmnop', + sourceCellId: 'production-gce-c7', + sourceCellIncarnation: cellIncarnation, + sourceAssignmentEpoch: 7, + graceMs: 60_000 +} + +describe('regional host drain endpoint', () => { + it('accepts only the dedicated identity and exact cell generation', async () => { + const drainHost = vi.fn(() => 'accepted' as const) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost, + cellIncarnation, + ready: vi.fn(async () => true) + }) + + const accepted = await post(app, 'rehome-token', request) + expect(accepted.status).toBe(200) + expect(await accepted.json()).toEqual({ v: 1, outcome: 'accepted' }) + expect(drainHost).toHaveBeenCalledWith(request) + + expect((await post(app, 'deploy-token', request)).status).toBe(401) + expect( + (await post(app, 'rehome-token', { + ...request, + sourceCellIncarnation: '33333333-3333-4333-8333-333333333333' + })).status + ).toBe(409) + expect(drainHost).toHaveBeenCalledOnce() + }) + + it('rejects malformed identities before touching the session registry', async () => { + const drainHost = vi.fn(() => 'accepted' as const) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost, + cellIncarnation, + ready: vi.fn(async () => true) + }) + + expect( + (await post(app, 'rehome-token', { ...request, relayHostId: 'raw-host-id' })).status + ).toBe(400) + expect(drainHost).not.toHaveBeenCalled() + }) + + it('is unavailable on the director', async () => { + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost: vi.fn(() => 'accepted' as const), + cellIncarnation, + ready: vi.fn(async () => true) + }) + expect((await post(app, 'rehome-token', request)).status).toBe(404) + }) + + it('proves the shared runtime identity is rejected without touching a session', async () => { + const drainHost = vi.fn(() => 'host-not-connected' as const) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost, + regionalRehomeTrustProbeHostExists: vi.fn(() => false), + regionalRehomeIdentityToken: vi.fn(async () => 'runtime-token'), + cellIncarnation, + ready: vi.fn(async () => true) + }) + const probe = { + ...request, + attemptId: REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + userId: REGIONAL_REHOME_TRUST_PROBE_USER_ID, + relayHostId: REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + sourceAssignmentEpoch: 1, + graceMs: 0 + } + + expect(REGIONAL_REHOME_TRUST_PROBE_HOST_ID).toHaveLength(16) + for (let call = 0; call < 2; call++) { + const response = await post(app, 'rehome-token', probe) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + v: 1, + outcome: 'host-not-connected', + sharedRuntimeIdentityRejected: true + }) + } + expect(drainHost).toHaveBeenCalledTimes(2) + }) + + it('fails closed if the synthetic host is not provably absent', async () => { + const drainHost = vi.fn(() => 'host-not-connected' as const) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost, + regionalRehomeTrustProbeHostExists: vi.fn(() => true), + regionalRehomeIdentityToken: vi.fn(async () => 'runtime-token'), + cellIncarnation, + ready: vi.fn(async () => true) + }) + const response = await post(app, 'rehome-token', { + ...request, + attemptId: REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + userId: REGIONAL_REHOME_TRUST_PROBE_USER_ID, + relayHostId: REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + sourceAssignmentEpoch: 1, + graceMs: 0 + }) + + expect(response.status).toBe(409) + expect(drainHost).not.toHaveBeenCalled() + }) + + it('rechecks synthetic host absence after asynchronous identity proof', async () => { + let hostExists = false + const drainHost = vi.fn(() => 'host-not-connected' as const) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost, + regionalRehomeTrustProbeHostExists: vi.fn(() => hostExists), + regionalRehomeIdentityToken: vi.fn(async () => { + hostExists = true + return 'runtime-token' + }), + cellIncarnation, + ready: vi.fn(async () => true) + }) + const response = await post(app, 'rehome-token', { + ...request, + attemptId: REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + userId: REGIONAL_REHOME_TRUST_PROBE_USER_ID, + relayHostId: REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + sourceAssignmentEpoch: 1, + graceMs: 0 + }) + + expect(response.status).toBe(409) + expect(drainHost).not.toHaveBeenCalled() + }) + + it('fails closed when the cell cannot prove its runtime identity rejection', async () => { + const drainHost = vi.fn(() => 'host-not-connected' as const) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost, + regionalRehomeTrustProbeHostExists: vi.fn(() => false), + regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'), + cellIncarnation, + ready: vi.fn(async () => true) + }) + const response = await post(app, 'rehome-token', { + ...request, + attemptId: REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + userId: REGIONAL_REHOME_TRUST_PROBE_USER_ID, + relayHostId: REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + sourceAssignmentEpoch: 1, + graceMs: 0 + }) + + expect(response.status).toBe(409) + expect(drainHost).not.toHaveBeenCalled() + }) +}) + +describe('regional rehome director controls', () => { + it('records cell capability separately from the legacy heartbeat contract', async () => { + const recordCellRegionalRehomeStatus = vi.fn().mockResolvedValue(undefined) + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: { recordCellRegionalRehomeStatus } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + const body = { + v: 1, + cellId: 'production-gce-c7', + cellIncarnation, + regionalRehomeProtocol: 1, + safety: { + observedAt: 100, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + } + const response = await postPath( + app, + '/v1/admin/cell-rehome-status', + 'runtime-token', + body + ) + + expect(response.status).toBe(200) + expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body) + }) + + it('accepts the exact safety payload the cell heartbeat composes', async () => { + // Why: index.ts spreads the FULL pool-pressure counts into safety; a + // strict schema missing any produced field 400s every heartbeat (the + // 2026-08-15..26 outage that left all cells at rehome protocol 0). + const recordCellRegionalRehomeStatus = vi.fn().mockResolvedValue(undefined) + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: { recordCellRegionalRehomeStatus } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + const observability = new RelayObservability( + { role: 'cell', cellId: 'production-gce-c7', region: 'us-central1' }, + () => {} + ) + observability.stop() + const body = { + v: 1, + cellId: 'production-gce-c7', + cellIncarnation, + regionalRehomeProtocol: 1, + safety: { + ...observability.regionalRehomeRuntimeSafety(), + ...emptyPostgresPoolPressureCounts() + } + } + const response = await postPath( + app, + '/v1/admin/cell-rehome-status', + 'runtime-token', + body + ) + + expect(response.status).toBe(200) + expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body) + }) + + it('applies the durable switch only with matching confirmation', async () => { + const inspectRegionalRehomeControl = vi.fn().mockResolvedValue({ + generation: 0, + enabled: false + }) + const applyRegionalRehomeControl = vi.fn(async (input) => ({ + ...input, + generation: input.expectedGeneration + 1 + })) + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: { + inspectRegionalRehomeControl, + applyRegionalRehomeControl + } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + expect((await postPath( + app, + '/v1/admin/regional-rehome-control', + 'deploy-token', + { v: 1, action: 'inspect' } + )).status).toBe(200) + const apply = { + v: 1, + action: 'apply', + expectedGeneration: 0, + enabled: true, + notBefore: 100, + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + drainGraceMs: 60_000, + confirmation: 'ENABLE_REGIONAL_REHOMING' + } + expect((await postPath( + app, + '/v1/admin/regional-rehome-control', + 'deploy-token', + apply + )).status).toBe(200) + expect(applyRegionalRehomeControl).toHaveBeenCalledOnce() + expect((await postPath( + app, + '/v1/admin/regional-rehome-control', + 'monitor-token', + { v: 1, action: 'inspect' } + )).status).toBe(200) + expect((await postPath( + app, + '/v1/admin/regional-rehome-control', + 'monitor-token', + apply + )).status).toBe(403) + expect((await postPath( + app, + '/v1/admin/regional-rehome-control', + 'deploy-token', + { ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' } + )).status).toBe(400) + }) + + it('probes dedicated trust twice and returns only aggregate proof', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + const cellDeploymentStatus = vi.fn().mockResolvedValue({ + cellId: 'production-gce-c7', + cellUrl: 'https://c7.relay.example.test', + region: 'us-central1', + runtime: { + cellIncarnation, + ready: true, + heartbeatFresh: true, + regionalRehomeProtocol: 1 + } + }) + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: { cellDeploymentStatus } as never, + drain: vi.fn(), + regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'), + regionalRehomeFetch: (async (url, init) => { + requests.push({ url: String(url), init }) + return Response.json({ + v: 1, + outcome: 'host-not-connected', + sharedRuntimeIdentityRejected: true + }) + }) as typeof fetch, + ready: vi.fn(async () => true) + }) + const response = await postPath( + app, + '/v1/admin/regional-rehome-trust-probe', + 'deploy-token', + { v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation } + ) + + expect(response.status).toBe(200) + const responseBody = await response.json() + expect(responseBody).toEqual({ + v: 1, + dedicatedIdentity: { + accepted: true, + firstOutcome: 'host-not-connected', + secondOutcome: 'host-not-connected', + idempotent: true + }, + sharedRuntimeIdentityRejected: true, + proven: true + }) + expect(requests).toHaveLength(2) + expect(requests.map(({ url }) => url)).toEqual([ + 'https://c7.relay.example.test/v1/admin/host-drain', + 'https://c7.relay.example.test/v1/admin/host-drain' + ]) + const bodies = requests.map(({ init }) => JSON.parse(String(init?.body))) + expect(bodies[0]).toEqual(bodies[1]) + expect(bodies[0]).toMatchObject({ + attemptId: REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + userId: REGIONAL_REHOME_TRUST_PROBE_USER_ID, + relayHostId: REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + sourceAssignmentEpoch: 1, + graceMs: 0 + }) + expect(requests.every(({ init }) => init?.signal instanceof AbortSignal)).toBe(true) + expect(JSON.stringify(responseBody)).not.toContain('rehome-token') + }) + + it('restricts trust probes to deploy authorization and strict input', async () => { + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + const body = { + v: 1, + sourceCellId: 'production-gce-c7', + sourceCellIncarnation: cellIncarnation + } + expect((await postPath( + app, + '/v1/admin/regional-rehome-trust-probe', + 'monitor-token', + body + )).status).toBe(401) + expect((await postPath( + app, + '/v1/admin/regional-rehome-trust-probe', + 'deploy-token', + { ...body, unexpected: true } + )).status).toBe(400) + }) + + it('fails closed when the source rejects the dedicated identity', async () => { + const cellDeploymentStatus = vi.fn().mockResolvedValue({ + cellUrl: 'https://c7.relay.example.test', + region: 'us-central1', + runtime: { + cellIncarnation, + ready: true, + heartbeatFresh: true, + regionalRehomeProtocol: 1 + } + }) + const sourceFetch = vi.fn().mockResolvedValue( + Response.json({ error: 'invalid_token' }, { status: 401 }) + ) + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: { cellDeploymentStatus } as never, + drain: vi.fn(), + regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'), + regionalRehomeFetch: sourceFetch, + ready: vi.fn(async () => true) + }) + const response = await postPath( + app, + '/v1/admin/regional-rehome-trust-probe', + 'deploy-token', + { v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation } + ) + + expect(response.status).toBe(409) + expect(sourceFetch).toHaveBeenCalledOnce() + expect(JSON.stringify(await response.json())).not.toContain('rehome-token') + }) +}) + +async function post( + app: ReturnType, + token: string, + body: unknown +): Promise { + return await app.request('/v1/admin/host-drain', { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(body) + }) +} + +async function postPath( + app: ReturnType, + path: string, + token: string, + body: unknown +): Promise { + return await app.request(path, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(body) + }) +} + +function config(overrides: Partial = {}): RelayConfig { + return { + port: 8080, + publicUrl: 'https://c7.relay.example.test', + cellUrl: 'https://c7.relay.example.test', + region: 'us-central1', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new Uint8Array(32), + role: 'cell', + cellId: 'production-gce-c7', + cells: [], + adminAudience: 'https://relay.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + rehomeDirectorServiceAccount: 'relay-director@example.test', + rehomeAudience: 'https://relay.example.test/v1/admin/host-drain', + runtimeServiceAccount: 'relay-cell@example.test', + adminJwksUrl: 'https://auth.example.test/jwks', + databasePoolMax: 10, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './data', + ...overrides + } +} diff --git a/cloud/apps/relay/src/regional-rehome-postgres.test.ts b/cloud/apps/relay/src/regional-rehome-postgres.test.ts new file mode 100644 index 00000000000..d36e26ecd68 --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-postgres.test.ts @@ -0,0 +1,552 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' +import { + REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT, + REGIONAL_REHOME_SQL_FAILURES_LIMIT, + REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT +} from './regional-rehome-safety.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip + +describePostgres('PostgreSQL regional rehoming', () => { + let primary: RelayDatabase + let secondary: RelayDatabase + let sequence = 0 + + beforeAll(async () => { + primary = await openRelayDatabase({ databaseUrl, dataDir: '' }) + secondary = await openRelayDatabase({ databaseUrl, dataDir: '' }) + }) + + beforeEach(async () => await cleanup()) + + afterAll(async () => { + await cleanup() + await secondary.close() + await primary.close() + }) + + async function cleanup(): Promise { + await primary.query( + `DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'` + ) + await primary.query(`DELETE FROM relay_region_rehome_worker_state`) + await primary.query(`DELETE FROM relay_region_rehome_control`) + await primary.query( + `DELETE FROM relay_control_connection_reservations + WHERE user_id LIKE 'pg-rehome-user-%'` + ) + await primary.query( + `DELETE FROM relay_assignment_migration_incarnations + WHERE user_id LIKE 'pg-rehome-user-%'` + ) + await primary.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id LIKE 'pg-rehome-user-%'` + ) + await primary.query( + `DELETE FROM relay_assignment_migrations WHERE user_id LIKE 'pg-rehome-user-%'` + ) + await primary.query( + `DELETE FROM relay_assignment_region_preferences + WHERE user_id LIKE 'pg-rehome-user-%'` + ) + await primary.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'pg-rehome-user-%'`) + for (const table of [ + 'relay_cell_rehome_safety', + 'relay_cell_capabilities', + 'relay_cell_connection_snapshots', + 'relay_cell_connection_runtime', + 'relay_cell_runtime', + 'relay_cell_connection_limits', + 'relay_cell_admission', + 'relay_cell_regions', + 'relay_cells' + ]) { + await primary.query(`DELETE FROM ${table} WHERE cell_id LIKE 'pg-rehome-cell-%'`) + } + } + + it('claims through ambient per-cell sql retry noise', async () => { + const context = await fixture() + await primary.query( + `UPDATE relay_cell_rehome_safety + SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT} + WHERE cell_id IN (?, ?)`, + [context.source.id, context.target.id] + ) + + expect(await context.store.claimRegionalRehome()).not.toBeNull() + }) + + it('skips an unclean cell without latching the control off', async () => { + const context = await fixture() + await primary.query( + `UPDATE relay_cell_rehome_safety + SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1} + WHERE cell_id = ?`, + [context.target.id] + ) + + expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 1, + enabled: true + }) + expect(await primary.query( + `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` + )).toEqual([{ next_dispatch_at: String(context.now() + 6_000) }]) + }) + + it('lets only one director claim a host', async () => { + const context = await fixture() + const claims = await Promise.all([ + context.store.claimRegionalRehome(), + context.competingStore.claimRegionalRehome() + ]) + + expect(claims.filter(Boolean)).toHaveLength(1) + expect(await primary.query( + `SELECT COUNT(*) AS count FROM relay_region_rehome_attempts + WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ count: '1' }]) + expect(await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations + WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`, + [context.identity.userId] + )).toEqual([{ count: '1' }]) + }) + + it('increments the disable generation once across competing directors', async () => { + const context = await fixture() + const disabled = await Promise.all([ + context.store.disableRegionalRehomeControl(), + context.competingStore.disableRegionalRehomeControl() + ]) + + expect(disabled.sort()).toEqual([false, true]) + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + }) + + it('records one receipt across competing directors', async () => { + const context = await fixture() + const attempt = await context.store.claimRegionalRehome() + const receipts = await Promise.all([ + context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted'), + context.competingStore.recordRegionalRehomeDrainReceipt( + attempt!.attemptId, + 'accepted' + ) + ]) + + expect(receipts.sort()).toEqual([false, true]) + expect(await primary.query( + `SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [attempt!.attemptId] + )).toEqual([{ drain_outcome: 'accepted' }]) + }) + + it('rechecks a preference changed while the assignment row is locked', async () => { + const context = await fixture() + let unlock!: () => void + let locked!: () => void + const lockedPromise = new Promise((resolve) => (locked = resolve)) + const unlockPromise = new Promise((resolve) => (unlock = resolve)) + const held = secondary.transaction(async (transaction) => { + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [context.identity.userId, context.identity.relayHostId] + ) + locked() + await unlockPromise + }) + await lockedPromise + const claim = context.store.claimRegionalRehome() + await primary.query( + `UPDATE relay_assignment_region_preferences SET preferred_region = 'us-central1', + observed_at = ? WHERE user_id = ? AND relay_host_id = ?`, + [context.now(), context.identity.userId, context.identity.relayHostId] + ) + unlock() + await held + + await expect(claim).resolves.toBeNull() + expect(await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ count: '0' }]) + }) + + it('rechecks fleet safety under locks before mutating a candidate', async () => { + const context = await fixture() + let unlock!: () => void + let locked!: () => void + const lockedPromise = new Promise((resolve) => (locked = resolve)) + const unlockPromise = new Promise((resolve) => (unlock = resolve)) + const held = secondary.transaction(async (transaction) => { + await transaction.queryLocked( + `SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, + [context.target.id] + ) + await transaction.query( + `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, + [context.target.id] + ) + locked() + await unlockPromise + }) + await lockedPromise + const claim = context.store.claimRegionalRehome() + unlock() + await held + + await expect(claim).resolves.toBeNull() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + expect(await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ count: '0' }]) + }) + + it('pauses when one required cell exceeds the reconnect limit', async () => { + const context = await fixture() + await primary.query( + `UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, + [REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, context.source.id] + ) + + await expect(context.store.claimRegionalRehome()).resolves.toBeNull() + await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({ + generation: 2, + enabled: false + }) + expect(await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ count: '0' }]) + }) + + it('does not retry a drain against a replacement source incarnation', async () => { + const context = await fixture() + const attempt = await context.store.claimRegionalRehome() + context.advance(31_000) + await heartbeat( + context.store, + context.source, + '33333333-3333-4333-8333-333333333333', + 1, + context.now() + ) + + await expect(context.competingStore.claimRegionalRehome()).resolves.toBeNull() + expect(await primary.query( + `SELECT send_attempts FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [attempt!.attemptId] + )).toEqual([{ send_attempts: '1' }]) + }) + + it('makes concurrent completion and expiry cleanup idempotent', async () => { + const context = await fixture() + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const targetControl = await context.store.activateControl(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(context.identity, context.sourceControl) + context.advance(24 * 60 * 60_000) + await heartbeat( + context.store, + context.source, + '11111111-1111-4111-8111-111111111111', + 1, + 900_000, + 2 + ) + await heartbeat( + context.store, + context.target, + '22222222-2222-4222-8222-222222222222', + 0, + 900_000, + 2 + ) + await context.store.renewControlActivity(context.identity, { + activityId: targetControl, + cellId: context.target.id, + expiresAt: context.now() + 90_000 + }) + + const outcomes = await Promise.all([ + context.store.completeReadyRegionalRehomes(), + context.competingStore.abortExpiredRegionalRehomes() + ]) + expect(outcomes).toEqual(expect.arrayContaining([0, 1])) + expect(await primary.query( + `SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted + FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ completed: true, aborted: false }]) + }) + + it('will not complete against a replacement target incarnation', async () => { + const context = await fixture() + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(context.identity, context.sourceControl) + context.advance(1) + await heartbeat( + context.store, + context.target, + '44444444-4444-4444-8444-444444444444', + 0, + context.now() + ) + + await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0) + expect(await primary.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ completed_at: null, aborted_at: null }]) + }) + + it('does not roll an unregistered target back to a stale regional source', async () => { + const context = await fixture() + await context.store.claimRegionalRehome() + context.advance(6 * 60_000) + await heartbeat( + context.store, + context.target, + '22222222-2222-4222-8222-222222222222', + 0, + 900_000, + 2 + ) + + await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0) + await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0) + expect(await primary.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }]) + }) + + it('completes after the drained host re-resolves through the director', async () => { + const context = await fixture() + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + // The drain recovery lands while both controls are still live. + await context.store.assign(context.identity, 'asia-east2') + expect(await controlAccounting(context.identity)).toEqual({ + reservedControls: 2, + controlLeases: 2 + }) + + await context.store.activateControl(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(context.identity, context.sourceControl) + + await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1) + expect(await primary.query( + `SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations + WHERE user_id = ?`, + [context.identity.userId] + )).toEqual([{ completed: true }]) + expect(await controlAccounting(context.identity)).toEqual({ + reservedControls: 1, + controlLeases: 1 + }) + }) + + it('repairs a skewed control counter before completing the rehome', async () => { + const context = await fixture() + const attempt = await context.store.claimRegionalRehome() + await context.store.activateControl(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(context.identity, { + cellId: context.target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(context.identity, context.sourceControl) + // Damage already written by a pre-fix sticky grant. + await primary.query( + `UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, + [context.identity.userId] + ) + + await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1) + expect(await controlAccounting(context.identity)).toEqual({ + reservedControls: 1, + controlLeases: 1 + }) + }) + + async function controlAccounting(identity: { + userId: string + relayHostId: string + }): Promise<{ reservedControls: number; controlLeases: number }> { + const assignment = ( + await primary.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0]! + const leases = await primary.query( + `SELECT COUNT(*) AS controls FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`, + [identity.userId, identity.relayHostId] + ) + return { + reservedControls: Number(assignment.reserved_controls), + controlLeases: Number(leases[0]!.controls) + } + } + + async function fixture() { + sequence++ + let now = 1_000_000 + const suffix = String(sequence) + const source = cell(suffix, 'source', 'us-central1') + const target = cell(suffix, 'target', 'asia-east2') + const store = new RelayAssignmentStore(primary, () => now, storeOptions) + const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions) + await store.inspectRegionalRehomeControl() + now += 24 * 60 * 60_000 + await store.applyRegionalRehomeControl({ + expectedGeneration: 0, + enabled: true, + notBefore: now, + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + drainGraceMs: 60_000 + }) + await store.reconcileCells([source, target]) + await heartbeat( + store, + source, + '11111111-1111-4111-8111-111111111111', + 1, + 900_000 + ) + await heartbeat( + store, + target, + '22222222-2222-4222-8222-222222222222', + 0, + 900_000 + ) + const identity = { + userId: `pg-rehome-user-${suffix}`, + relayHostId: `rehomehost${suffix.padStart(6, '0')}` + } + const assignment = await store.assign(identity, undefined, 'us-central1') + const sourceControl = await store.activateControl(identity, { + cellId: source.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.assign(identity, 'asia-east2') + return { + store, + competingStore, + identity, + source, + target, + sourceControl, + now: () => now, + advance: (milliseconds: number) => { + now += milliseconds + } + } + } +}) + +const storeOptions = { + requireLiveCells: true, + heartbeatTtlMs: 45_000 +} + +function cell(suffix: string, role: string, region: 'us-central1' | 'asia-east2') { + return { + id: `pg-rehome-cell-${suffix}-${role}`, + url: `https://pg-rehome-${suffix}-${role}.example.test`, + region, + capacityRequests: 100, + connectionHardCap: 1_000 as const, + connectionUnobservedBound: 60 + } +} + +async function heartbeat( + store: RelayAssignmentStore, + cellConfig: ReturnType, + cellIncarnation: string, + regionalRehomeProtocol: number, + startedAt: number, + connectionInclusionWatermark = 1 +): Promise { + await store.recordCellHeartbeat({ + cellId: cellConfig.id, + cellUrl: cellConfig.url, + region: cellConfig.region, + cellIncarnation, + startedAt, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionInclusionWatermark, + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + }) + await store.recordCellRegionalRehomeStatus({ + cellId: cellConfig.id, + cellIncarnation, + regionalRehomeProtocol, + safety: { + observedAt: 1_000_000 + 24 * 60 * 60_000, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + }) +} diff --git a/cloud/apps/relay/src/regional-rehome-safety.test.ts b/cloud/apps/relay/src/regional-rehome-safety.test.ts new file mode 100644 index 00000000000..5c7d10d37a0 --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-safety.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from 'vitest' +import { + REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT, + REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT, + REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT, + REGIONAL_REHOME_SQL_FAILURES_LIMIT, + regionalRehomeSafetyFailure +} from './regional-rehome-safety.js' + +const NOW = 1_787_900_000_000 + +function safety(overrides: Partial[0]> = {}) { + return { + observedAt: NOW - 1_000, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0, + ...overrides + } +} + +describe('regionalRehomeSafetyFailure', () => { + it('passes the measured healthy-fleet baseline of pool micro-waits', () => { + // Every production cell idles at 1-2 peak waiters resolved in ~1ms; a + // zero-tolerance bar here disables the worker on its first tick. + expect( + regionalRehomeSafetyFailure( + safety({ databasePoolWaitersMax: 2, databasePoolWaitMsMax: 1 }), + NOW, + 19 + ) + ).toBeNull() + }) + + it('passes routine client reconnect churn', () => { + expect( + regionalRehomeSafetyFailure(safety({ reconnects: 19 * 80 }), NOW, 19) + ).toBeNull() + }) + + it('still fails closed on each pool pressure bound', () => { + for (const overrides of [ + { databasePoolWaitersMax: REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT + 1 }, + { databasePoolWaitMsMax: REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT + 1 } + ]) { + expect(regionalRehomeSafetyFailure(safety(overrides), NOW, 19)).toBe( + 'database_pool_pressure' + ) + } + }) + + it('still fails closed on a reconnect storm', () => { + expect( + regionalRehomeSafetyFailure( + safety({ reconnects: 19 * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1 }), + NOW, + 19 + ) + ).toBe('elevated_reconnects') + }) + + it('passes ambient 55P03 retry noise', () => { + // Fleet-wide retried lock timeouts peaked at 82 counted failures per + // minute over Aug 25-28; the combined snapshot can span two windows. + expect(regionalRehomeSafetyFailure(safety({ sqlFailures: 164 }), NOW, 19)).toBeNull() + }) + + it('still fails closed on a sql failure storm', () => { + expect( + regionalRehomeSafetyFailure( + safety({ sqlFailures: REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1 }), + NOW, + 19 + ) + ).toBe('sql_failures') + // Literal storm magnitude (measured 2026-08-28) pins the bar itself: the + // limit must sit below real storm scale, not merely exist. + expect(regionalRehomeSafetyFailure(safety({ sqlFailures: 395 }), NOW, 19)).toBe( + 'sql_failures' + ) + // The bar is exclusive: exactly at the limit still passes. + expect( + regionalRehomeSafetyFailure( + safety({ sqlFailures: REGIONAL_REHOME_SQL_FAILURES_LIMIT }), + NOW, + 19 + ) + ).toBeNull() + }) + + it('keeps zero-tolerance for real failure signals', () => { + expect( + regionalRehomeSafetyFailure( + safety({ controlActivityRecoveryFailures: 1 }), + NOW, + 19 + ) + ).toBe('control_recovery_failures') + expect(regionalRehomeSafetyFailure(safety({ observedAt: 0 }), NOW, 19)).toBe( + 'monitoring_stale' + ) + expect( + regionalRehomeSafetyFailure(safety({ observedAt: NOW - 61_000 }), NOW, 19) + ).toBe('monitoring_stale') + }) +}) diff --git a/cloud/apps/relay/src/regional-rehome-safety.ts b/cloud/apps/relay/src/regional-rehome-safety.ts new file mode 100644 index 00000000000..4da51f71b0d --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-safety.ts @@ -0,0 +1,86 @@ +import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' + +// Limits sit well above the healthy-fleet baseline measured in production on +// 2026-08-28 (peak 2 waiters / 1ms pool waits on every cell; up to ~80 +// reconnects per published two-window row on the busiest cell). Sustained +// pool saturation still trips: waiters-max 16 is 8x baseline yet far under a +// backed-up pool, and 250ms peak wait is 1/10 of the incident-monitor alert. +// Instantaneous databasePoolWaiting is not checked separately: it is bounded +// by databasePoolWaitersMax within every published window. +export const REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT = 250 +export const REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT = 16 +export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250 + +// Counted SQL failures are dominated by relay_cells 55P03 lock-timeout +// retries the transaction wrapper heals in place (Aug 25-28: ~1000 retried +// attempts per day vs ~30 exhausted, those clustered in one storm), so a +// zero bar disabled the worker on ambient noise just like the original pool +// bars. Fleet-wide ambient noise peaked at 82 counted failures per minute +// over four days; genuine database distress produced 395-457. The combined +// snapshot spans up to two 30s windows per process (pathological ambient +// alignment ~164), so 250 stays clear of noise while storms still trip. +// Terminal outages also trip the pool bars and the worker's own +// dispatch-failure budget; the sql bar only needs to catch storms. +export const REGIONAL_REHOME_SQL_FAILURES_LIMIT = 250 +// Per-cell candidate cleanliness is a soft skip, not a durable latch; the +// worst ambient per-cell publish carried ~24 counted failures (two windows +// of 12). The fleet bar deliberately dominates: cells that are individually +// clean can sum past 250, and a fleet-wide sum at that scale is a storm. +export const REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT = 40 + +export function regionalRehomePoolPressure(safety: { + databasePoolWaitersMax: number + databasePoolWaitMsMax: number +}): boolean { + return ( + safety.databasePoolWaitersMax > REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT || + safety.databasePoolWaitMsMax > REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT + ) +} + +export function regionalRehomeSafetyFailure( + safety: RegionalRehomeSafetySnapshot, + now: number, + requiredCells: number +): string | null { + if (safety.observedAt === 0 || now - safety.observedAt > 60_000) { + return 'monitoring_stale' + } + if (safety.sqlFailures > REGIONAL_REHOME_SQL_FAILURES_LIMIT) return 'sql_failures' + if (regionalRehomePoolPressure(safety)) { + return 'database_pool_pressure' + } + if (safety.controlActivityRecoveryFailures > 0) { + return 'control_recovery_failures' + } + const reconnectLimit = + Math.max(1, requiredCells) * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + if (safety.reconnects > reconnectLimit) return 'elevated_reconnects' + return null +} + +export function combineRegionalRehomeSafety( + processSafety: RegionalRehomeSafetySnapshot, + fleetSafety: RegionalRehomeSafetySnapshot +): RegionalRehomeSafetySnapshot { + return { + observedAt: Math.min(processSafety.observedAt, fleetSafety.observedAt), + sqlFailures: processSafety.sqlFailures + fleetSafety.sqlFailures, + reconnects: fleetSafety.reconnects, + controlActivityRecoveryFailures: + processSafety.controlActivityRecoveryFailures + + fleetSafety.controlActivityRecoveryFailures, + databasePoolWaiting: Math.max( + processSafety.databasePoolWaiting, + fleetSafety.databasePoolWaiting + ), + databasePoolWaitersMax: Math.max( + processSafety.databasePoolWaitersMax, + fleetSafety.databasePoolWaitersMax + ), + databasePoolWaitMsMax: Math.max( + processSafety.databasePoolWaitMsMax, + fleetSafety.databasePoolWaitMsMax + ) + } +} diff --git a/cloud/apps/relay/src/regional-rehome-store.test.ts b/cloud/apps/relay/src/regional-rehome-store.test.ts new file mode 100644 index 00000000000..43f293b1131 --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-store.test.ts @@ -0,0 +1,1399 @@ +import { describe, expect, it } from 'vitest' +import { + RelayAssignmentStore, + REGIONAL_REHOME_QUARANTINE_FAILURES, + REGIONAL_REHOME_QUARANTINE_MS, + REGIONAL_REHOME_REDRAIN_SEND_LIMIT +} from './assignment-store.js' +import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js' +import { + REGIONAL_REHOME_SQL_FAILURES_LIMIT, + REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT +} from './regional-rehome-safety.js' +import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' + +const source = { + id: 'us-c1', + url: 'https://us-c1.relay.example.test', + region: 'us-central1' as const, + capacityRequests: 100, + connectionHardCap: 1_000 as const, + connectionUnobservedBound: 60 +} +const target = { + id: 'asia-c1', + url: 'https://asia-c1.relay.example.test', + region: 'asia-east2' as const, + capacityRequests: 100, + connectionHardCap: 1_000 as const, + connectionUnobservedBound: 60 +} +const sourceIncarnation = '11111111-1111-4111-8111-111111111111' +const targetIncarnation = '22222222-2222-4222-8222-222222222222' + +describe('regional rehome assignment state', () => { + it('does not open a transaction while the worker is disabled', async () => { + const delegate = await openInMemoryRelayDatabase() + const database = new TransactionCountingDatabase(delegate) + const store = new RelayAssignmentStore(database, () => 1_000_000) + await store.inspectRegionalRehomeControl() + database.transactionCalls = 0 + + await expect(store.claimRegionalRehome()).resolves.toBeNull() + expect(database.transactionCalls).toBe(0) + await database.close() + }) + + it('initializes a missing control row without opening a transaction', async () => { + const delegate = await openInMemoryRelayDatabase() + const database = new TransactionCountingDatabase(delegate) + const store = new RelayAssignmentStore(database, () => 1_000_000) + + await expect(store.claimRegionalRehome()).resolves.toBeNull() + expect(database.transactionCalls).toBe(0) + await expect(store.inspectRegionalRehomeControl()).resolves.toMatchObject({ + generation: 0, + enabled: false + }) + await database.close() + }) + + it('uses a generation-bound durable kill switch', async () => { + const context = await setup() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 1, + enabled: true, + ratePerMinute: 10 + }) + expect(await context.store.disableRegionalRehomeControl()).toBe(true) + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + await expect(context.store.applyRegionalRehomeControl({ + expectedGeneration: 1, + enabled: true, + notBefore: context.now(), + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + drainGraceMs: 60_000 + })).rejects.toThrow('regional_rehome_generation_mismatch') + await expect(context.store.applyRegionalRehomeControl({ + expectedGeneration: 2, + enabled: true, + notBefore: context.now(), + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + drainGraceMs: 60_000 + })).resolves.toMatchObject({ generation: 3, enabled: true }) + await context.database.close() + }) + + it('moves one live preferred host and completes without disabling its source cell', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const neighbor = { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } + const sourceControl = await activatePreferredSource(context, identity) + await activateSource(context, neighbor) + + const attempt = await context.store.claimRegionalRehome() + expect(attempt).toMatchObject({ + userId: identity.userId, + relayHostId: identity.relayHostId, + sourceCellId: source.id, + sourceCellIncarnation: sourceIncarnation, + targetCellId: target.id, + targetCellIncarnation: targetIncarnation, + previousEpoch: 1, + assignmentEpoch: 2, + sendAttempts: 1 + }) + expect(await context.store.resolve(neighbor)).toMatchObject({ cellId: source.id }) + expect(await context.store.completeReadyRegionalRehomes()).toBe(0) + expect( + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + ).toBe(true) + expect( + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + ).toBe(false) + + const targetControl = await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + expect(await context.store.completeReadyRegionalRehomes()).toBe(0) + await context.store.releaseActivity(identity, sourceControl) + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + + expect(await context.store.resolve(identity)).toMatchObject({ + cellId: target.id, + assignmentEpoch: 2 + }) + expect(await context.store.resolve(neighbor)).toMatchObject({ cellId: source.id }) + expect(await context.database.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + )).toEqual([{ completed_at: context.now(), aborted_at: null }]) + expect(await context.database.query( + `SELECT completed_at, aborted_at FROM relay_region_rehome_attempts` + )).toEqual([{ completed_at: context.now(), aborted_at: null }]) + expect(targetControl).toMatch(/^control:/) + await context.database.close() + }) + + it('completes from durable activity when the drain response was lost', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + expect(await context.database.query( + `SELECT drain_receipt_at, completed_at, aborted_at + FROM relay_region_rehome_attempts` + )).toEqual([{ drain_receipt_at: null, completed_at: context.now(), aborted_at: null }]) + await context.database.close() + }) + + it('requires a fresh preference and an advertised source capability', async () => { + const context = await setup({ sourceProtocol: 0 }) + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) + await context.database.close() + }) + + it('fails fleet safety closed until source and target telemetry is fresh', async () => { + const context = await setup() + await context.database.query( + `DELETE FROM relay_cell_rehome_safety WHERE cell_id = ?`, + [target.id] + ) + expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({ + requiredCells: 2, + missingCells: 1, + observedAt: 0 + }) + await heartbeat(context.store, source, sourceIncarnation, 1, 2, { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 2, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + await heartbeat(context.store, target, targetIncarnation, 0, 2, { + observedAt: context.now(), + sqlFailures: 1, + reconnects: 3, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({ + requiredCells: 2, + missingCells: 0, + observedAt: context.now(), + sqlFailures: 1, + reconnects: 5 + }) + await context.database.close() + }) + + it('claims through the measured healthy baseline of pool micro-waits and churn', async () => { + const context = await setup() + const baseline = { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 42, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 2, + databasePoolWaitersMax: 2, + databasePoolWaitMsMax: 1 + } + await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline) + await heartbeat(context.store, target, targetIncarnation, 0, 2, baseline) + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + + expect(await context.store.claimRegionalRehome()).toMatchObject({ + sourceCellId: source.id, + targetCellId: target.id + }) + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + enabled: true + }) + await context.database.close() + }) + + it('latches off on a per-cell reconnect storm even when the fleet sum is low', async () => { + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + await context.database.query( + `UPDATE relay_cell_rehome_safety SET reconnects = 251 WHERE cell_id = ?`, + [source.id] + ) + + const warnings = collectDisableWarnings() + try { + expect(await context.store.claimRegionalRehome()).toBeNull() + } finally { + warnings.restore() + } + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + expect(warnings.entries).toMatchObject([ + { reason: 'elevated_reconnects', maxReconnects: 251, controlGeneration: 2 } + ]) + await context.database.close() + }) + + it('latches off on sustained pool pressure and logs the disable exactly once', async () => { + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + await context.database.query( + `UPDATE relay_cell_rehome_safety SET database_pool_waiters_max = 17 WHERE cell_id = ?`, + [target.id] + ) + + const warnings = collectDisableWarnings() + try { + expect(await context.store.claimRegionalRehome()).toBeNull() + // Already disabled: the next tick returns before the gate and stays silent. + expect(await context.store.claimRegionalRehome()).toBeNull() + } finally { + warnings.restore() + } + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + expect(warnings.entries).toMatchObject([ + { reason: 'database_pool_pressure', databasePoolWaitersMax: 17 } + ]) + await context.database.close() + }) + + it('claims through ambient per-cell sql retry noise', async () => { + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + await context.database.query( + `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT}` + ) + + expect(await context.store.claimRegionalRehome()).not.toBeNull() + await context.database.close() + }) + + it('skips an unclean cell without latching the control off', async () => { + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + await activatePreferredSource(context, { + userId: 'user-2', + relayHostId: 'ponmlkjihgfedcba' + }) + // Above the per-cell cleanliness bar but below the fleet storm bar: the + // candidate is skipped this tick while the worker stays enabled. + await context.database.query( + `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1} WHERE cell_id = ?`, + [target.id] + ) + + const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') + try { + expect(await context.store.claimRegionalRehome()).toBeNull() + } finally { + warnings.restore() + } + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 1, + enabled: true + }) + // The skip is visible and named, and both candidates blocked by the one + // unclean cell accumulate into a single entry. + expect(warnings.entries).toMatchObject([ + { + skips: [ + { + reason: 'target_unclean', + cellId: target.id, + sqlFailures: REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1, + candidates: 2 + } + ] + } + ]) + // A skipped tick is charged the dispatch interval: candidate scans stay + // rate-limited even when nothing claims. + expect( + await context.database.query( + `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` + ) + ).toEqual([{ next_dispatch_at: context.now() + 6_000 }]) + expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) + await context.database.close() + }) + + it('does not throttle or log an idle tick with no candidates', async () => { + const context = await setup() + + const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') + try { + expect(await context.store.claimRegionalRehome()).toBeNull() + } finally { + warnings.restore() + } + expect(warnings.entries).toEqual([]) + expect( + await context.database.query( + `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` + ) + ).toEqual([{ next_dispatch_at: 0 }]) + await context.database.close() + }) + + it('atomically latches durable control off when candidate safety changes', async () => { + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + await context.database.query( + `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, + [target.id] + ) + + expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) + await context.database.close() + }) + + it('rechecks locked fleet safety before retrying a drain dispatch', async () => { + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + expect(await context.store.claimRegionalRehome()).not.toBeNull() + context.advance(31_000) + await context.database.query( + `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, + [target.id] + ) + + expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + await context.database.close() + }) + + it('latches off after three dispatch failures and resumes only through CAS', async () => { + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + const first = await context.store.claimRegionalRehome() + for (let index = 0; index < 3; index++) { + await context.store.recordRegionalRehomeDispatchFailure(first!.attemptId) + } + context.advance(5 * 60_000 - 1) + expect(await context.store.claimRegionalRehome()).toBeNull() + context.advance(1) + await heartbeat(context.store, source, sourceIncarnation, 1, 2, { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + await heartbeat(context.store, target, targetIncarnation, 0, 2, { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 2, + enabled: false + }) + await context.store.applyRegionalRehomeControl({ + expectedGeneration: 2, + enabled: true, + notBefore: context.now(), + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + drainGraceMs: 60_000 + }) + const retry = await context.store.claimRegionalRehome() + expect(retry).toMatchObject({ attemptId: first!.attemptId, sendAttempts: 2 }) + expect(await context.database.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations` + )).toEqual([{ count: 1 }]) + await context.database.close() + }) + + it('refreshes only the migration leases while source splices drain', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + await context.store.acquireActivity(identity, { + activityId: 'splice:source', + kind: 'splice', + cellId: source.id + }) + await context.store.claimRegionalRehome() + const before = await context.database.query( + `SELECT activity_id, expires_at FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`, + [identity.userId, identity.relayHostId] + ) + context.advance(60_000) + expect(await context.store.refreshRegionalRehomeLeases()).toBe(1) + const after = await context.database.query( + `SELECT activity_id, expires_at FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`, + [identity.userId, identity.relayHostId] + ) + const beforeById = new Map(before.map((row) => [row.activity_id, row.expires_at])) + const afterById = new Map(after.map((row) => [row.activity_id, row.expires_at])) + expect(Number(afterById.get('migration:2'))).toBeGreaterThan( + Number(beforeById.get('migration:2')) + ) + expect(Number(afterById.get('control-pending:2'))).toBeGreaterThan( + Number(beforeById.get('control-pending:2')) + ) + expect(afterById.get('splice:source')).toBe(beforeById.get('splice:source')) + await context.database.close() + }) + + it('stops refreshing an unregistered target and lets normal rollback retire it', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + await context.store.claimRegionalRehome() + context.advance(6 * 60_000) + expect(await context.store.refreshRegionalRehomeLeases()).toBe(0) + await heartbeat(context.store, source, sourceIncarnation, 1, 2) + expect(await context.store.abortExpiredEvacuations()).toBe(1) + expect(await context.store.reapRegionalRehomeAttempts()).toBe(1) + expect(await context.store.resolve(identity)).toMatchObject({ + cellId: source.id, + assignmentEpoch: 3 + }) + expect(await context.database.query( + `SELECT completed_at, aborted_at FROM relay_region_rehome_attempts` + )).toEqual([{ completed_at: null, aborted_at: context.now() }]) + await context.database.close() + }) + + it('does not roll an unregistered target back to a stale regional source', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + await context.store.claimRegionalRehome() + context.advance(6 * 60_000) + await heartbeat(context.store, target, targetIncarnation, 0, 2) + + expect(await context.store.refreshRegionalRehomeLeases()).toBe(0) + expect(await context.store.abortExpiredEvacuations()).toBe(0) + expect( + await context.database.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ cell_id: target.id, assignment_epoch: 2 }]) + await context.database.close() + }) + + it('rolls back an inactive registered target only after the 24-hour bound', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt( + attempt!.attemptId, + 'accepted' + ) + const targetControl = await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.store.releaseActivity(identity, sourceControl) + await context.store.releaseActivity(identity, targetControl) + context.advance(24 * 60 * 60_000) + await heartbeat(context.store, source, sourceIncarnation, 1, 2) + expect(await context.store.abortExpiredRegionalRehomes()).toBe(1) + expect(await context.store.resolve(identity)).toMatchObject({ + cellId: source.id, + assignmentEpoch: 3 + }) + await context.database.close() + }) + + it('redrains a receipted dual-homed attempt once its grace elapses', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + + // Before grace elapses a receipted attempt is not re-dispatched. + context.advance(30 * 60_000) + await freshHeartbeats(context) + expect(await context.store.claimRegionalRehome()).toBeNull() + + context.advance(30 * 60_000 + 1) + await freshHeartbeats(context) + const redrain = await context.store.claimRegionalRehome() + expect(redrain).toMatchObject({ + attemptId: attempt!.attemptId, + drainGraceMs: 0, + sendAttempts: 2 + }) + // The per-dispatch receipt replaces the original without a mismatch. + await expect( + context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'host-not-connected') + ).resolves.toBe(true) + + // Redrains are spaced: nothing new inside the redrain interval. + context.advance(30_000) + await freshHeartbeats(context) + expect(await context.store.claimRegionalRehome()).toBeNull() + context.advance(30_001) + await freshHeartbeats(context) + expect(await context.store.claimRegionalRehome()).toMatchObject({ + attemptId: attempt!.attemptId, + drainGraceMs: 0, + sendAttempts: 3 + }) + + // Once the host actually leaves the source, completion wins over redrain. + await context.store.releaseActivity(identity, sourceControl) + context.advance(60_001) + await freshHeartbeats(context) + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 2 + }) + expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + await context.database.close() + }) + + it('resets the failure budget on a repeated redrain receipt outcome', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) + await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) + + context.advance(60 * 60_000 + 1) + await freshHeartbeats(context) + expect(await context.store.claimRegionalRehome()).toMatchObject({ + attemptId: attempt!.attemptId, + drainGraceMs: 0 + }) + // The repeated outcome still proves the source answered. + expect( + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + ).toBe(false) + await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + generation: 1, + enabled: true + }) + await context.database.close() + }) + + it('does not redrain before the target registers or when the fleet is unsafe', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + + // Past grace but the target never registered: force-closing the source + // would disconnect the host with nowhere proven to land. + context.advance(60 * 60_000 + 1) + await freshHeartbeats(context) + expect(await context.store.claimRegionalRehome()).toBeNull() + + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.database.query( + `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, + [target.id] + ) + expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ + enabled: false + }) + await context.database.close() + }) + + it('completes healthy candidates past a poisoned attempt and logs it', async () => { + const context = await setup() + const poisoned = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const healthy = { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } + const poisonedSource = await activatePreferredSource(context, poisoned) + const healthySource = await activatePreferredSource(context, healthy) + const first = await context.store.claimRegionalRehome() + context.advance(6_000) + const second = await context.store.claimRegionalRehome() + expect(first!.userId).toBe(poisoned.userId) + expect(second!.userId).toBe(healthy.userId) + for (const [identity, attempt, sourceControl] of [ + [poisoned, first, poisonedSource], + [healthy, second, healthySource] + ] as const) { + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + } + // The production poison shape: the assignment moved past the attempt. + await context.database.query( + `UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 5 + WHERE user_id = ?`, + [poisoned.userId] + ) + const warnings = collectCandidateFailureWarnings() + try { + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + } finally { + warnings.restore() + } + expect(warnings.entries).toEqual([ + { + event: 'orca_relay_regional_rehome_candidate_failed', + operation: 'complete', + attemptId: first!.attemptId, + reason: 'regional_rehome_assignment_mismatch' + } + ]) + expect(await context.database.query( + `SELECT completed_at FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [second!.attemptId] + )).toEqual([{ completed_at: context.now() }]) + await context.database.close() + }) + + it('quarantines a repeatedly failing candidate and redacts free-form errors', async () => { + const context = await setup() + const poisoned = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const source1 = await activatePreferredSource(context, poisoned) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(poisoned, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(poisoned, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(poisoned, source1) + await context.database.query( + `UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 5 + WHERE user_id = ?`, + [poisoned.userId] + ) + const warnings = collectCandidateFailureWarnings() + try { + for (let round = 0; round < REGIONAL_REHOME_QUARANTINE_FAILURES; round++) { + expect(await context.store.completeReadyRegionalRehomes()).toBe(0) + } + expect(warnings.entries).toHaveLength(REGIONAL_REHOME_QUARANTINE_FAILURES) + // Quarantined: the poisoned row leaves the candidate page entirely. + expect(await context.store.completeReadyRegionalRehomes()).toBe(0) + expect(warnings.entries).toHaveLength(REGIONAL_REHOME_QUARANTINE_FAILURES) + // After the quarantine window it is retried (and fails) once more. + context.advance(REGIONAL_REHOME_QUARANTINE_MS + 1) + await context.store.completeReadyRegionalRehomes() + expect(warnings.entries).toHaveLength(REGIONAL_REHOME_QUARANTINE_FAILURES + 1) + // A free-form error (never a slug) reaches the log only as 'redacted'. + expect( + warnings.entries.every( + (entry) => entry.reason === 'regional_rehome_assignment_mismatch' + ) + ).toBe(true) + context.advance(REGIONAL_REHOME_QUARANTINE_MS + 1) + const database = context.database + const original = database.transaction.bind(database) + database.transaction = () => { + throw new Error('postgresql://secret@database.invalid/relay') + } + try { + await context.store.completeReadyRegionalRehomes() + } finally { + database.transaction = original + } + const last = warnings.entries.at(-1)! + expect(last.reason).toBe('redacted') + expect(JSON.stringify(last)).not.toContain('secret') + } finally { + warnings.restore() + } + await context.database.close() + }) + + it('aborts healthy expired candidates past a poisoned attempt', async () => { + const context = await setup() + const poisoned = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const healthy = { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } + const poisonedSource = await activatePreferredSource(context, poisoned) + const healthySource = await activatePreferredSource(context, healthy) + const first = await context.store.claimRegionalRehome() + context.advance(6_000) + const second = await context.store.claimRegionalRehome() + for (const [identity, attempt, sourceControl] of [ + [poisoned, first, poisonedSource], + [healthy, second, healthySource] + ] as const) { + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const targetControl = await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + await context.store.releaseActivity(identity, targetControl) + } + await context.database.query( + `UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 5 + WHERE user_id = ?`, + [poisoned.userId] + ) + context.advance(24 * 60 * 60_000) + await freshHeartbeats(context) + const warnings = collectCandidateFailureWarnings() + try { + expect(await context.store.abortExpiredRegionalRehomes()).toBe(1) + } finally { + warnings.restore() + } + expect(warnings.entries).toEqual([ + { + event: 'orca_relay_regional_rehome_candidate_failed', + operation: 'abort', + attemptId: first!.attemptId, + reason: 'regional_rehome_assignment_mismatch' + } + ]) + expect(await context.database.query( + `SELECT aborted_at FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [second!.attemptId] + )).toEqual([{ aborted_at: context.now() }]) + await context.database.close() + }) + + it('keeps dual-control accounting when the drained host re-resolves mid-rehome', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + expect(await controlAccounting(context, identity)).toEqual({ + reservedControls: 2, + controlLeases: 2 + }) + + // The drained host re-resolves through the director while both the source + // control and the target's pending control are still live. + await context.store.assign(identity, 'asia-east2') + expect(await controlAccounting(context, identity)).toEqual({ + reservedControls: 2, + controlLeases: 2 + }) + + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + expect(await controlAccounting(context, identity)).toEqual({ + reservedControls: 1, + controlLeases: 1 + }) + + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + expect(await context.database.query( + `SELECT completed_at FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + )).toEqual([{ completed_at: context.now() }]) + expect(await cellReservations(context)).toEqual({ [source.id]: 0, [target.id]: 1 }) + await context.database.close() + }) + + it('grants a host whose counter was skewed without duplicating its control', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.releaseActivity(identity, sourceControl) + // Damage already written by a pre-fix sticky grant. + await context.database.query( + `UPDATE relay_assignments SET reserved_controls = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + + await context.store.assign(identity, 'asia-east2') + expect(await context.database.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`, + [identity.userId, identity.relayHostId] + )).toEqual([{ activity_id: `control:${target.id}:1` }]) + expect(await cellReservations(context)).toEqual({ [source.id]: 0, [target.id]: 2 }) + await context.database.close() + }) + + it('repairs a skewed control counter before completing the rehome', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + // Damage already written by a pre-fix sticky grant. + await context.database.query( + `UPDATE relay_assignments SET reserved_controls = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + expect(await controlAccounting(context, identity)).toEqual({ + reservedControls: 1, + controlLeases: 1 + }) + expect(await context.database.query( + `SELECT migration_leases FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + )).toEqual([{ migration_leases: 0 }]) + await context.database.close() + }) + + it('reports a repaired counter only for the candidate it repaired', async () => { + const context = await setup() + const skewed = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const clean = { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } + const skewedSource = await activatePreferredSource(context, skewed) + const cleanSource = await activatePreferredSource(context, clean) + const first = await context.store.claimRegionalRehome() + context.advance(6_000) + const second = await context.store.claimRegionalRehome() + for (const [identity, attempt, sourceControl] of [ + [skewed, first, skewedSource], + [clean, second, cleanSource] + ] as const) { + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + } + // Damage already written by a pre-fix sticky grant, on one host only. + await context.database.query( + `UPDATE relay_assignments SET reserved_controls = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [skewed.userId, skewed.relayHostId] + ) + + const warnings = collectCandidateFailureWarnings([ + 'orca_relay_regional_rehome_activity_counts_repaired' + ]) + try { + expect(await context.store.completeReadyRegionalRehomes()).toBe(2) + } finally { + warnings.restore() + } + expect(warnings.entries).toEqual([ + { + event: 'orca_relay_regional_rehome_activity_counts_repaired', + attemptId: first!.attemptId + } + ]) + await context.database.close() + }) + + it('never repairs past a migration lease whose shape is wrong', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + await context.database.query( + `UPDATE relay_assignments SET reserved_controls = 0 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + await context.database.query( + `UPDATE relay_assignment_migrations SET target_reserved_units = 9 + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + + const warnings = collectCandidateFailureWarnings() + try { + expect(await context.store.completeReadyRegionalRehomes()).toBe(0) + } finally { + warnings.restore() + } + expect(warnings.entries).toEqual([ + { + event: 'orca_relay_regional_rehome_candidate_failed', + operation: 'complete', + attemptId: attempt!.attemptId, + reason: 'migration_activity_lease_shape_mismatch' + } + ]) + expect(await controlAccounting(context, identity)).toEqual({ + reservedControls: 0, + controlLeases: 1 + }) + await context.database.close() + }) + + it('stays silent when a repair cannot make the counts whole', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: attempt!.assignmentEpoch + }) + await context.store.releaseActivity(identity, sourceControl) + // A vanished migration lease is repairable arithmetic on the first assert + // but still wrong on the re-assert: no repaired event may leak out. + await context.database.query( + `DELETE FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'migration'`, + [identity.userId, identity.relayHostId] + ) + + const warnings = collectCandidateFailureWarnings([ + 'orca_relay_regional_rehome_candidate_failed', + 'orca_relay_regional_rehome_activity_counts_repaired' + ]) + try { + expect(await context.store.completeReadyRegionalRehomes()).toBe(0) + } finally { + warnings.restore() + } + expect(warnings.entries).toEqual([ + { + event: 'orca_relay_regional_rehome_candidate_failed', + operation: 'complete', + attemptId: attempt!.attemptId, + reason: 'migration_activity_accounting_mismatch' + } + ]) + await context.database.close() + }) + + it('caps redrain dispatches at the send limit', async () => { + const context = await setup() + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.database.query( + `UPDATE relay_region_rehome_attempts SET send_attempts = ? WHERE attempt_id = ?`, + [REGIONAL_REHOME_REDRAIN_SEND_LIMIT, attempt!.attemptId] + ) + context.advance(60 * 60_000 + 1) + await freshHeartbeats(context) + expect(await context.store.claimRegionalRehome()).toBeNull() + await context.database.close() + }) +}) + +class TransactionCountingDatabase implements RelayDatabase { + transactionCalls = 0 + + constructor(private readonly delegate: RelayDatabase) {} + + query(sql: string, params?: unknown[]): Promise { + return this.delegate.query(sql, params) + } + + queryLocked( + sql: string, + params?: unknown[], + options?: { failIfUnavailable?: boolean } + ): Promise { + return this.delegate.queryLocked(sql, params, options) + } + + transaction( + operation: (transaction: RelayDatabase) => Promise, + options?: { reportRetries?: boolean } + ): Promise { + this.transactionCalls += 1 + return this.delegate.transaction(operation, options) + } + + close(): Promise { + return this.delegate.close() + } +} + +type Context = Awaited> + +function collectDisableWarnings() { + const entries: Record[] = [] + const original = console.warn + console.warn = (line: unknown, ...rest: unknown[]) => { + try { + const parsed = JSON.parse(line as string) as Record + if (parsed.event === 'orca_relay_regional_rehome_safety_disabled') { + entries.push(parsed) + return + } + } catch { + // fall through to the real console for non-JSON lines + } + original(line, ...rest) + } + return { + entries, + restore: () => { + console.warn = original + } + } +} + +async function setup(options: { sourceProtocol?: number } = {}) { + let clock = 1_000_000 + const database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, () => clock, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.inspectRegionalRehomeControl() + clock += 24 * 60 * 60_000 + await store.applyRegionalRehomeControl({ + expectedGeneration: 0, + enabled: true, + notBefore: clock, + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + drainGraceMs: 60 * 60_000 + }) + await store.reconcileCells([source, target]) + await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1) + await heartbeat(store, target, targetIncarnation, 0) + return { + database, + store, + now: () => clock, + advance: (milliseconds: number) => { + clock += milliseconds + } + } +} + +function collectEventWarnings(event: string) { + const entries: Record[] = [] + const original = console.warn + console.warn = (line: unknown, ...rest: unknown[]) => { + try { + const parsed = JSON.parse(line as string) as Record + if (parsed.event === event) { + entries.push(parsed) + return + } + } catch { + // fall through to the real console for non-JSON lines + } + original(line, ...rest) + } + return { + entries, + restore: () => { + console.warn = original + } + } +} + +function collectCandidateFailureWarnings( + events: string[] = ['orca_relay_regional_rehome_candidate_failed'] +) { + const entries: Record[] = [] + const original = console.warn + console.warn = (line: unknown, ...rest: unknown[]) => { + try { + const parsed = JSON.parse(line as string) as Record + if (events.includes(parsed.event as string)) { + entries.push(parsed) + return + } + } catch { + // fall through to the real console for non-JSON lines + } + original(line, ...rest) + } + return { + entries, + restore: () => { + console.warn = original + } + } +} + +async function controlAccounting( + context: Context, + identity: { userId: string; relayHostId: string } +): Promise<{ reservedControls: number; controlLeases: number }> { + const assignment = ( + await context.database.query( + `SELECT reserved_controls FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0]! + const leases = await context.database.query( + `SELECT COUNT(*) AS controls FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`, + [identity.userId, identity.relayHostId] + ) + return { + reservedControls: Number(assignment.reserved_controls), + controlLeases: Number(leases[0]!.controls) + } +} + +async function cellReservations(context: Context): Promise> { + const rows = await context.database.query( + `SELECT cell_id, reserved_requests FROM relay_cells ORDER BY cell_id` + ) + return Object.fromEntries( + rows.map((row) => [String(row.cell_id), Number(row.reserved_requests)]) + ) +} + +async function freshHeartbeats(context: Context): Promise { + const safety = { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + // The clock doubles as a strictly-increasing connection inclusion watermark. + await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety) + await heartbeat(context.store, target, targetIncarnation, 0, context.now(), safety) +} + +async function activatePreferredSource( + context: Context, + identity: { userId: string; relayHostId: string } +): Promise { + const assignment = await context.store.assign(identity, undefined, 'us-central1') + const control = await context.store.activateControl(identity, { + cellId: source.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await context.store.assign(identity, 'asia-east2') + return control +} + +async function activateSource( + context: Context, + identity: { userId: string; relayHostId: string } +): Promise { + const assignment = await context.store.assign(identity, undefined, 'us-central1') + return await context.store.activateControl(identity, { + cellId: source.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) +} + +async function heartbeat( + store: RelayAssignmentStore, + cell: typeof source | typeof target, + cellIncarnation: string, + regionalRehomeProtocol: number, + connectionInclusionWatermark = 1, + regionalRehomeSafety?: RegionalRehomeSafetySnapshot +): Promise { + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + region: cell.region, + cellIncarnation, + startedAt: 900_000, + ready: true, + observedRequests: 0, + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + connectionInclusionWatermark, + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + }) + await store.recordCellRegionalRehomeStatus({ + cellId: cell.id, + cellIncarnation, + regionalRehomeProtocol, + safety: regionalRehomeSafety ?? { + observedAt: 1_000_000 + 24 * 60 * 60_000, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + }) +} diff --git a/cloud/apps/relay/src/regional-rehome-target-selection.test.ts b/cloud/apps/relay/src/regional-rehome-target-selection.test.ts new file mode 100644 index 00000000000..e2190168735 --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-target-selection.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openInMemoryRelayDatabase } from './database.js' +import { REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT } from './regional-rehome-safety.js' + +const cell = (id: string, region: 'us-central1' | 'asia-east2') => ({ + id, + url: `https://${id}.relay.example.test`, + region, + capacityRequests: 100, + connectionHardCap: 1_000 as const, + connectionUnobservedBound: 60 +}) +const source = cell('us-c1', 'us-central1') +const noHeadroom = cell('asia-a', 'asia-east2') +const unclean = cell('asia-b', 'asia-east2') +const highLoad = cell('asia-c', 'asia-east2') +const lowLoad = cell('asia-d', 'asia-east2') + +const incarnation = (n: number) => + `${String(n).repeat(8)}-${String(n).repeat(4)}-4${String(n).repeat(3)}` + + `-8${String(n).repeat(3)}-${String(n).repeat(12)}` + +async function setup() { + let clock = 1_000_000 + const database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, () => clock, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.inspectRegionalRehomeControl() + clock += 24 * 60 * 60_000 + await store.applyRegionalRehomeControl({ + expectedGeneration: 0, + enabled: true, + notBefore: clock, + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + drainGraceMs: 60 * 60_000 + }) + await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad]) + const beat = async ( + config: typeof source, + n: number, + protocol: number, + state: { observedRequests: number; enforcedConnections: number; sqlFailures: number } + ) => { + await store.recordCellHeartbeat({ + cellId: config.id, + cellUrl: config.url, + region: config.region, + cellIncarnation: incarnation(n), + startedAt: 900_000, + ready: true, + observedRequests: state.observedRequests, + totalConnections: state.enforcedConnections, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: state.enforcedConnections, + connectionInclusionWatermark: clock, + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + }) + await store.recordCellRegionalRehomeStatus({ + cellId: config.id, + cellIncarnation: incarnation(n), + regionalRehomeProtocol: protocol, + safety: { + observedAt: clock, + sqlFailures: state.sqlFailures, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + }) + } + const activatePreferredSource = async () => { + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const assignment = await store.assign(identity, undefined, 'us-central1') + await store.activateControl(identity, { + cellId: source.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.assign(identity, 'asia-east2') + } + return { database, store, beat, activatePreferredSource } +} + +const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1 + +describe('regional rehome target selection', () => { + it('never selects a target without connection headroom, even at lowest load', async () => { + const context = await setup() + await context.beat(source, 1, 1, { + observedRequests: 0, + enforcedConnections: 0, + sqlFailures: 0 + }) + // Lowest load but the connection hard cap is exhausted. + await context.beat(noHeadroom, 2, 0, { + observedRequests: 0, + enforcedConnections: 999, + sqlFailures: 0 + }) + await context.beat(unclean, 3, 0, { + observedRequests: 0, + enforcedConnections: 0, + sqlFailures: UNCLEAN + }) + await context.beat(highLoad, 4, 0, { + observedRequests: 50, + enforcedConnections: 0, + sqlFailures: 0 + }) + await context.beat(lowLoad, 5, 0, { + observedRequests: 10, + enforcedConnections: 0, + sqlFailures: 0 + }) + await context.activatePreferredSource() + + const attempt = await context.store.claimRegionalRehome() + expect(attempt?.targetCellId).toBe(lowLoad.id) + await context.database.close() + }) + + it('falls to the next clean target when the load winner goes unclean', async () => { + const context = await setup() + await context.beat(source, 1, 1, { + observedRequests: 0, + enforcedConnections: 0, + sqlFailures: 0 + }) + await context.beat(noHeadroom, 2, 0, { + observedRequests: 0, + enforcedConnections: 999, + sqlFailures: 0 + }) + await context.beat(unclean, 3, 0, { + observedRequests: 0, + enforcedConnections: 0, + sqlFailures: UNCLEAN + }) + await context.beat(highLoad, 4, 0, { + observedRequests: 50, + enforcedConnections: 0, + sqlFailures: 0 + }) + await context.beat(lowLoad, 5, 0, { + observedRequests: 10, + enforcedConnections: 0, + sqlFailures: UNCLEAN + }) + await context.activatePreferredSource() + + const attempt = await context.store.claimRegionalRehome() + expect(attempt?.targetCellId).toBe(highLoad.id) + await context.database.close() + }) +}) diff --git a/cloud/apps/relay/src/regional-rehome-trust-probe.ts b/cloud/apps/relay/src/regional-rehome-trust-probe.ts new file mode 100644 index 00000000000..57ae2468886 --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-trust-probe.ts @@ -0,0 +1,107 @@ +import { z } from 'zod' + +export const REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID = + '00000000-0000-4000-8000-000000000001' +export const REGIONAL_REHOME_TRUST_PROBE_USER_ID = 'regional-rehome-trust-probe' +export const REGIONAL_REHOME_TRUST_PROBE_HOST_ID = 'trustprobe000001' + +const SOURCE_PROBE_TIMEOUT_MS = 10_000 + +const SourceProbeResponseSchema = z + .object({ + v: z.literal(1), + outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected']), + sharedRuntimeIdentityRejected: z.literal(true) + }) + .strict() + +type SourceProbeOutcome = z.infer['outcome'] + +export type RegionalRehomeTrustProbeResult = { + v: 1 + dedicatedIdentity: { + accepted: boolean + firstOutcome: SourceProbeOutcome + secondOutcome: SourceProbeOutcome + idempotent: boolean + } + sharedRuntimeIdentityRejected: boolean + proven: boolean +} + +export async function probeRegionalRehomeTrust(input: { + sourceCellUrl: string + sourceCellId: string + sourceCellIncarnation: string + audience: string + identityToken: (audience: string) => Promise + fetch: typeof fetch +}): Promise { + const token = await input.identityToken(input.audience) + const body = JSON.stringify({ + v: 1, + attemptId: REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID, + userId: REGIONAL_REHOME_TRUST_PROBE_USER_ID, + relayHostId: REGIONAL_REHOME_TRUST_PROBE_HOST_ID, + sourceCellId: input.sourceCellId, + sourceCellIncarnation: input.sourceCellIncarnation, + sourceAssignmentEpoch: 1, + graceMs: 0 + }) + const call = async (): Promise> => { + const response = await input.fetch( + new URL('/v1/admin/host-drain', input.sourceCellUrl), + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body, + signal: AbortSignal.timeout(SOURCE_PROBE_TIMEOUT_MS) + } + ) + if (!response.ok) throw new Error(`regional_rehome_trust_probe_source_${response.status}`) + const parsed = SourceProbeResponseSchema.safeParse(await response.json()) + if (!parsed.success) throw new Error('regional_rehome_trust_probe_source_invalid_response') + return parsed.data + } + const first = await call() + const second = await call() + const idempotent = first.outcome === second.outcome + const sharedRuntimeIdentityRejected = + first.sharedRuntimeIdentityRejected && second.sharedRuntimeIdentityRejected + const proven = + first.outcome === 'host-not-connected' && + second.outcome === 'host-not-connected' && + idempotent && + sharedRuntimeIdentityRejected + if (!proven) throw new Error('regional_rehome_trust_probe_not_proven') + return { + v: 1, + dedicatedIdentity: { + accepted: true, + firstOutcome: first.outcome, + secondOutcome: second.outcome, + idempotent + }, + sharedRuntimeIdentityRejected, + proven + } +} + +export function isRegionalRehomeTrustProbe(input: { + attemptId: string + userId: string + relayHostId: string + sourceAssignmentEpoch: number + graceMs: number +}): boolean { + return ( + input.attemptId === REGIONAL_REHOME_TRUST_PROBE_ATTEMPT_ID && + input.userId === REGIONAL_REHOME_TRUST_PROBE_USER_ID && + input.relayHostId === REGIONAL_REHOME_TRUST_PROBE_HOST_ID && + input.sourceAssignmentEpoch === 1 && + input.graceMs === 0 + ) +} diff --git a/cloud/apps/relay/src/regional-rehome-worker.test.ts b/cloud/apps/relay/src/regional-rehome-worker.test.ts new file mode 100644 index 00000000000..af905bb9ab2 --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-worker.test.ts @@ -0,0 +1,227 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import { + combineRegionalRehomeSafety, + REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT, + regionalRehomeSafetyFailure +} from './regional-rehome-safety.js' +import { startRegionalRehomeWorker } from './regional-rehome-worker.js' + +describe('regional rehome worker', () => { + afterEach(() => vi.restoreAllMocks()) + + it('sends an incarnation- and source-epoch-bound drain without exposing identity', async () => { + let now = 0 + const attempt = { + attemptId: '11111111-1111-4111-8111-111111111111', + userId: 'private-user', + relayHostId: 'abcdefghijklmnop', + preferredRegion: 'asia-east2', + sourceCellId: 'production-gce-c7', + sourceCellUrl: 'https://c7.relay.example.test', + sourceCellIncarnation: '22222222-2222-4222-8222-222222222222', + targetCellId: 'production-gce-c27', + targetCellIncarnation: '33333333-3333-4333-8333-333333333333', + previousEpoch: 7, + assignmentEpoch: 8, + drainGraceMs: 60_000, + sendAttempts: 1 + } + const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt) + const recordRegionalRehomeDrainReceipt = vi.fn().mockResolvedValue(true) + const recordRegionalRehomeWorkerFailure = vi.fn().mockResolvedValue(undefined) + const assignments = { + claimRegionalRehome, + recordRegionalRehomeDrainReceipt, + recordRegionalRehomeWorkerFailure + } as unknown as RelayAssignmentStore + const requests: Array<{ url: string; init?: RequestInit }> = [] + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const worker = startRegionalRehomeWorker(config(), assignments, { + now: () => now, + safetySnapshot: () => safety(now), + intervalMs: 60_000, + identityToken: async (audience) => { + expect(audience).toBe('https://relay.example.test/v1/admin/host-drain') + return 'secret-token' + }, + fetch: (async (url, init) => { + requests.push({ url: String(url), init }) + return Response.json({ v: 1, outcome: 'accepted' }) + }) as typeof fetch + })! + await settleWorker() + now = 1_000 + await worker.run() + worker.stop() + + expect(requests).toHaveLength(1) + expect(requests[0]!.url).toBe('https://c7.relay.example.test/v1/admin/host-drain') + expect(requests[0]!.url).not.toContain('secret-token') + expect(requests[0]!.init?.headers).toMatchObject({ + authorization: 'Bearer secret-token' + }) + expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({ + v: 1, + attemptId: '11111111-1111-4111-8111-111111111111', + userId: 'private-user', + relayHostId: 'abcdefghijklmnop', + sourceCellId: 'production-gce-c7', + sourceCellIncarnation: '22222222-2222-4222-8222-222222222222', + sourceAssignmentEpoch: 7, + graceMs: 60_000 + }) + expect(recordRegionalRehomeDrainReceipt).toHaveBeenCalledWith( + '11111111-1111-4111-8111-111111111111', + 'accepted' + ) + const logs = warn.mock.calls.map((call) => String(call[0])).join('\n') + expect(logs).not.toContain('private-user') + expect(logs).not.toContain('abcdefghijklmnop') + }) + + it('fails closed before the observation gate and records bounded dispatch failures', async () => { + let now = 0 + const attempt = { + attemptId: '11111111-1111-4111-8111-111111111111', + userId: 'private-user', + relayHostId: 'abcdefghijklmnop', + sourceCellId: 'source', + sourceCellUrl: 'https://source.example.test', + sourceCellIncarnation: '22222222-2222-4222-8222-222222222222', + targetCellId: 'target', + previousEpoch: 1, + assignmentEpoch: 2, + drainGraceMs: 60_000, + sendAttempts: 1 + } + const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt) + const assignments = { + claimRegionalRehome, + recordRegionalRehomeDispatchFailure: vi.fn().mockResolvedValue(undefined), + recordRegionalRehomeWorkerFailure: vi.fn().mockResolvedValue(undefined) + } as unknown as RelayAssignmentStore + const worker = startRegionalRehomeWorker(config(), assignments, { + now: () => now, + safetySnapshot: () => safety(now), + intervalMs: 60_000, + identityToken: async () => { + throw new Error('token unavailable') + } + })! + await settleWorker() + claimRegionalRehome.mockClear() + now = 100 + await worker.run() + worker.stop() + expect(assignments.recordRegionalRehomeDispatchFailure).toHaveBeenCalledWith( + '11111111-1111-4111-8111-111111111111' + ) + expect(assignments.recordRegionalRehomeWorkerFailure).not.toHaveBeenCalled() + }) + + it('passes unsafe process telemetry to the durable claim gate', async () => { + let now = 0 + let sqlFailures = 0 + const claimRegionalRehome = vi.fn().mockResolvedValue(null) + const assignments = { + claimRegionalRehome + } as unknown as RelayAssignmentStore + const worker = startRegionalRehomeWorker(config(), assignments, { + now: () => now, + safetySnapshot: () => ({ ...safety(now), sqlFailures }), + intervalMs: 60_000 + })! + await settleWorker() + claimRegionalRehome.mockClear() + now = 100 + sqlFailures = 1 + await worker.run() + worker.stop() + + expect(claimRegionalRehome).toHaveBeenCalledWith( + expect.objectContaining({ observedAt: 100, sqlFailures: 1 }) + ) + }) + + it('starts inert on directors so durable control can enable without a restart', async () => { + let now = 0 + const claimRegionalRehome = vi.fn().mockResolvedValue(null) + const assignments = { + claimRegionalRehome + } as unknown as RelayAssignmentStore + const worker = startRegionalRehomeWorker( + config(), + assignments, + { + now: () => now, + safetySnapshot: () => safety(now), + intervalMs: 60_000 + } + ) + expect(worker).not.toBeNull() + await settleWorker() + claimRegionalRehome.mockClear() + now = 100 + await worker!.run() + worker!.stop() + expect(claimRegionalRehome).toHaveBeenCalledOnce() + + expect( + startRegionalRehomeWorker( + config({ role: 'cell' }), + {} as RelayAssignmentStore, + { safetySnapshot: () => safety(1) } + ) + ).toBeNull() + }) + + it('treats the reconnect threshold as per-cell and excludes the director', () => { + const cells = 2 + const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + const processSafety = { ...safety(100), reconnects: limit * 10 } + const fleetSafety = { ...safety(100), reconnects: limit } + expect(regionalRehomeSafetyFailure( + combineRegionalRehomeSafety(processSafety, fleetSafety), + 100, + cells + )).toBeNull() + expect(regionalRehomeSafetyFailure( + combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }), + 100, + cells + )).toBe('elevated_reconnects') + }) +}) + +function config(overrides: Partial = {}): RelayConfig { + return { + role: 'director', + rehomeAudience: 'https://relay.example.test/v1/admin/host-drain', + rehomeDirectorServiceAccount: 'relay-director@example.test', + ...overrides + } as RelayConfig +} + +function safety(observedAt: number) { + return { + requiredCells: 2, + missingCells: 0, + observedAt, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolTotal: 3, + databasePoolIdle: 3, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolOldestWaitMs: 0, + databasePoolWaitMsMax: 0 + } +} + +async function settleWorker(): Promise { + await Promise.resolve() + await Promise.resolve() +} diff --git a/cloud/apps/relay/src/regional-rehome-worker.ts b/cloud/apps/relay/src/regional-rehome-worker.ts new file mode 100644 index 00000000000..97c63a61025 --- /dev/null +++ b/cloud/apps/relay/src/regional-rehome-worker.ts @@ -0,0 +1,122 @@ +import { z } from 'zod' +import type { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import { googleMetadataIdentityToken } from './google-metadata-identity-token.js' +import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' + +type RegionalRehomeWorkerOptions = { + fetch?: typeof fetch + identityToken?: (audience: string) => Promise + now?: () => number + intervalMs?: number + requestTimeoutMs?: number + safetySnapshot?: () => RegionalRehomeSafetySnapshot +} + +export type RegionalRehomeWorker = { + run: () => Promise + stop: () => void +} + +const RegionalHostDrainResponseSchema = z + .object({ + v: z.literal(1), + outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected']) + }) + .strict() + +export function startRegionalRehomeWorker( + config: RelayConfig, + assignments: RelayAssignmentStore, + options: RegionalRehomeWorkerOptions = {} +): RegionalRehomeWorker | null { + if ( + config.role !== 'director' || + !config.rehomeAudience || + !config.rehomeDirectorServiceAccount || + !options.safetySnapshot + ) { + return null + } + const audience = config.rehomeAudience + const safetySnapshot = options.safetySnapshot + const now = options.now ?? Date.now + const fetchImpl = options.fetch ?? fetch + const tokenProvider = + options.identityToken ?? + ((audience: string) => googleMetadataIdentityToken(audience, fetchImpl)) + let stopped = false + let inFlight = false + const run = async (): Promise => { + if (stopped || inFlight) return + inFlight = true + let attemptId: string | null = null + try { + const processSafety = safetySnapshot() + const attempt = await assignments.claimRegionalRehome(processSafety) + if (!attempt) return + attemptId = attempt.attemptId + const token = await tokenProvider(audience) + const response = await fetchImpl( + new URL('/v1/admin/host-drain', attempt.sourceCellUrl), + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + v: 1, + attemptId: attempt.attemptId, + userId: attempt.userId, + relayHostId: attempt.relayHostId, + sourceCellId: attempt.sourceCellId, + sourceCellIncarnation: attempt.sourceCellIncarnation, + sourceAssignmentEpoch: attempt.previousEpoch, + graceMs: attempt.drainGraceMs + }), + signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000) + } + ) + if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`) + const body = RegionalHostDrainResponseSchema.safeParse(await response.json()) + if (!body.success) throw new Error('regional_rehome_source_invalid_response') + await assignments.recordRegionalRehomeDrainReceipt( + attempt.attemptId, + body.data.outcome + ) + console.warn( + JSON.stringify({ + event: 'orca_relay_regional_rehome_dispatched', + sourceCellId: attempt.sourceCellId, + targetCellId: attempt.targetCellId, + outcome: body.data.outcome, + sendAttempts: attempt.sendAttempts + }) + ) + } catch (error) { + await (attemptId + ? assignments.recordRegionalRehomeDispatchFailure(attemptId) + : assignments.recordRegionalRehomeWorkerFailure() + ).catch(() => undefined) + console.warn( + JSON.stringify({ + event: 'orca_relay_regional_rehome_dispatch_failed', + reason: error instanceof Error ? error.message : 'unknown' + }) + ) + } finally { + inFlight = false + } + } + const timer = setInterval(() => void run(), options.intervalMs ?? 1_000) + timer.unref() + void run() + return { + run, + stop: () => { + stopped = true + clearInterval(timer) + } + } +} diff --git a/cloud/apps/relay/src/registered-migration-abandonment.ts b/cloud/apps/relay/src/registered-migration-abandonment.ts new file mode 100644 index 00000000000..9127f935f2a --- /dev/null +++ b/cloud/apps/relay/src/registered-migration-abandonment.ts @@ -0,0 +1,81 @@ +export const REGISTERED_MIGRATION_ABANDON_MS = 24 * 60 * 60 * 1_000 + +export const DURABLY_FENCED_MIGRATION_SOURCE = `( + EXISTS ( + SELECT 1 FROM relay_cell_committed_fences committed_source_fence + JOIN relay_cell_fence_attempts source_fence_attempt + ON source_fence_attempt.attempt_id = committed_source_fence.attempt_id + JOIN relay_cell_runtime source_runtime + ON source_runtime.cell_id = committed_source_fence.cell_id + WHERE committed_source_fence.cell_id = migration.source_cell_id + AND source_fence_attempt.cell_id = committed_source_fence.cell_id + AND source_fence_attempt.cell_incarnation = committed_source_fence.cell_incarnation + AND source_fence_attempt.completed_at IS NOT NULL + AND source_fence_attempt.aborted_at IS NULL + AND source_runtime.cell_incarnation = committed_source_fence.cell_incarnation + AND committed_source_fence.attested_at >= source_runtime.last_heartbeat_at + ) + OR EXISTS ( + SELECT 1 FROM relay_cell_legacy_fence_adoptions adopted_source_fence + JOIN relay_cell_runtime source_runtime + ON source_runtime.cell_id = adopted_source_fence.cell_id + WHERE adopted_source_fence.cell_id = migration.source_cell_id + AND source_runtime.cell_incarnation = adopted_source_fence.cell_incarnation + AND adopted_source_fence.attested_at >= source_runtime.last_heartbeat_at + ) +)` + +// Ordinary recovery waits on admission age; a completed fence proves the source cannot return. +export const ABANDONED_REGISTERED_MIGRATION = `( + migration.target_registered_at IS NOT NULL + AND + migration.expires_at <= ? + AND + ( + EXISTS ( + SELECT 1 FROM relay_cells target_cell + JOIN relay_cell_admission target_admission + ON target_admission.cell_id = target_cell.cell_id + WHERE target_cell.cell_id = migration.target_cell_id + AND target_cell.enabled = 0 + AND target_admission.admission_state = 'existing-only' + AND target_admission.updated_at <= ? + ) + OR ( + EXISTS ( + SELECT 1 FROM relay_cells source_cell + JOIN relay_cell_admission source_admission + ON source_admission.cell_id = source_cell.cell_id + WHERE source_cell.cell_id = migration.source_cell_id + AND source_cell.enabled = 0 + AND source_admission.admission_state = 'existing-only' + AND ( + source_admission.updated_at <= ? + OR ${DURABLY_FENCED_MIGRATION_SOURCE} + ) + ) + AND EXISTS ( + SELECT 1 FROM relay_cells target_cell + JOIN relay_cell_admission target_admission + ON target_admission.cell_id = target_cell.cell_id + WHERE target_cell.cell_id = migration.target_cell_id + AND target_cell.enabled = 1 + AND target_admission.admission_state IN ('migration-only', 'general') + ) + AND NOT EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases source_activity + WHERE source_activity.user_id = migration.user_id + AND source_activity.relay_host_id = migration.relay_host_id + AND source_activity.cell_id = migration.source_cell_id + ) + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases target_control + WHERE target_control.user_id = migration.user_id + AND target_control.relay_host_id = migration.relay_host_id + AND target_control.cell_id = migration.target_cell_id + AND target_control.activity_kind = 'control' + AND target_control.activity_id NOT LIKE 'control-pending:%' + ) +)` diff --git a/cloud/apps/relay/src/registered-migration-inventory-postgres.test.ts b/cloud/apps/relay/src/registered-migration-inventory-postgres.test.ts new file mode 100644 index 00000000000..ad318b2ad3a --- /dev/null +++ b/cloud/apps/relay/src/registered-migration-inventory-postgres.test.ts @@ -0,0 +1,139 @@ +import pg from 'pg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { openRelayDatabase, type RelayDatabase } from './database.js' +import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js' +import { readRegisteredMigrationInventory } from './registered-migration-inventory.js' + +const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = databaseUrl ? describe : describe.skip +const schema = 'relay_migration_inventory_test' + +describePostgres('PostgreSQL registered migration inventory', () => { + let database: RelayDatabase | undefined + + beforeAll(async () => { + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + await client.query(`CREATE SCHEMA ${schema}`) + } finally { + await client.end() + } + const url = new URL(databaseUrl!) + url.searchParams.set('options', `-c search_path=${schema}`) + database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' }) + }) + + afterAll(async () => { + await database?.close() + const client = new pg.Client({ connectionString: databaseUrl }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + } finally { + await client.end() + } + }) + + it('counts a disabled-target migration even while its source is active', async () => { + const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000 + await database!.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES ('cell-a', 'https://a.example.test', 1, 4000, 1, 0, 0, 1), + ('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 1)` + ) + await database!.query( + `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at) + VALUES ('cell-a', 'general', 1), ('cell-b', 'existing-only', 1)` + ) + await database!.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`, + [now, now] + ) + await database!.query( + `INSERT INTO relay_assignment_activity_leases + (user_id, relay_host_id, activity_id, activity_kind, cell_id, + request_units, expires_at, updated_at) + VALUES ('user-1', '111111111111', 'control:source:1', 'control', 'cell-a', 1, ?, ?)`, + [now, now] + ) + await database!.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch, + assignment_epoch, source_request_units, target_reserved_units, expires_at, + target_registered_at, created_at, updated_at) + VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, 2, 2, 1, 2)` + ) + + expect(await readRegisteredMigrationInventory(database!, now)).toEqual({ + open: 1, + inactive: 1, + abandoned: 1, + pairs: [ + { + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + open: 1, + inactive: 1, + abandoned: 1 + } + ] + }) + + await database!.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES ('cell-c', 'https://c.example.test', 1, 4000, 0, 0, 0, 1)` + ) + await database!.query( + `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at) + VALUES ('cell-c', 'migration-only', 1)` + ) + await database!.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES ('user-2', '222222222222', 'cell-c', 2, ?, ?, 0, 0, 0, 0, 0, 0)`, + [now, now] + ) + await database!.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch, + assignment_epoch, source_request_units, target_reserved_units, expires_at, + target_registered_at, created_at, updated_at) + VALUES ('user-2', '222222222222', 'cell-a', 'cell-c', 1, 2, 1, 2, ?, NULL, 1, 2)`, + [now + 60_000] + ) + + expect(await readRegisteredMigrationInventory(database!, now)).toEqual({ + open: 2, + inactive: 1, + abandoned: 1, + pairs: [ + { + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + open: 1, + inactive: 1, + abandoned: 1 + }, + { + sourceCellId: 'cell-a', + targetCellId: 'cell-c', + open: 1, + inactive: 0, + abandoned: 0 + } + ] + }) + }) +}) diff --git a/cloud/apps/relay/src/registered-migration-inventory.test.ts b/cloud/apps/relay/src/registered-migration-inventory.test.ts new file mode 100644 index 00000000000..41aa26ac732 --- /dev/null +++ b/cloud/apps/relay/src/registered-migration-inventory.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from 'vitest' +import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js' +import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js' +import { + formatRegisteredMigrationInventory, + readRegisteredMigrationInventory +} from './registered-migration-inventory.js' + +describe('registered migration inventory', () => { + it('does not restart abandonment when an old source migration lease is refreshed', async () => { + const database = await openInMemoryRelayDatabase() + const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000 + await insertCell(database, 'cell-a', false, 'existing-only') + await insertCell(database, 'cell-b', true, 'migration-only') + await insertMigration(database, now - 1) + + const fresh = await readRegisteredMigrationInventory(database, now) + + expect(fresh).toEqual({ + open: 1, + inactive: 1, + abandoned: 1, + pairs: [ + { + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + open: 1, + inactive: 1, + abandoned: 1 + } + ] + }) + expect(formatRegisteredMigrationInventory(fresh)).toEqual([ + '[orca-relay] migration inventory open=1 expiredRegisteredInactive=1 abandonedRegistered=1', + '[orca-relay] migration pair sourceCellId=cell-a targetCellId=cell-b open=1 expiredRegisteredInactive=1 abandoned=1' + ]) + expect( + ( + await readRegisteredMigrationInventory( + database, + now + 1 + ) + ).abandoned + ).toBe(1) + }) + + it('includes unregistered open migrations without logging a host identity', async () => { + const database = await openInMemoryRelayDatabase() + const now = 10_000 + await insertCell(database, 'cell-a', false, 'existing-only') + await insertCell(database, 'cell-b', true, 'migration-only') + await insertMigration(database, now + 60_000, null) + + const inventory = await readRegisteredMigrationInventory(database, now) + + expect(inventory).toEqual({ + open: 1, + inactive: 0, + abandoned: 0, + pairs: [ + { + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + open: 1, + inactive: 0, + abandoned: 0 + } + ] + }) + expect(formatRegisteredMigrationInventory(inventory).join('\n')).not.toContain( + '111111111111' + ) + }) +}) + +async function insertCell( + database: RelayDatabase, + cellId: string, + enabled: boolean, + admissionState: string +): Promise { + await database.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES (?, ?, ?, 4000, 0, 0, 0, 1)`, + [cellId, `https://${cellId}.example.test`, enabled ? 1 : 0] + ) + await database.query( + `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at) + VALUES (?, ?, 1)`, + [cellId, admissionState] + ) +} + +async function insertMigration( + database: RelayDatabase, + expiresAt: number, + targetRegisteredAt: number | null = 2 +): Promise { + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, + last_activity_at, reserved_controls, reserved_splices, reserved_invites, + pending_installs, pending_confirmations, migration_leases) + VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`, + [expiresAt, expiresAt] + ) + await database.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch, + assignment_epoch, source_request_units, target_reserved_units, expires_at, + target_registered_at, created_at, updated_at) + VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, ?, ?, 1, 2)`, + [expiresAt, targetRegisteredAt] + ) +} diff --git a/cloud/apps/relay/src/registered-migration-inventory.ts b/cloud/apps/relay/src/registered-migration-inventory.ts new file mode 100644 index 00000000000..3d47ad31732 --- /dev/null +++ b/cloud/apps/relay/src/registered-migration-inventory.ts @@ -0,0 +1,104 @@ +import type { RelayDatabase, SqlRow } from './database.js' +import { + ABANDONED_REGISTERED_MIGRATION, + REGISTERED_MIGRATION_ABANDON_MS +} from './registered-migration-abandonment.js' + +export type RegisteredMigrationInventory = { + open: number + inactive: number + abandoned: number + pairs: Array<{ + sourceCellId: string + targetCellId: string + open: number + inactive: number + abandoned: number + }> +} + +export async function readRegisteredMigrationInventory( + database: RelayDatabase, + now: number +): Promise { + const abandonedBefore = now - REGISTERED_MIGRATION_ABANDON_MS + const openRows = await database.query( + `SELECT migration.source_cell_id, migration.target_cell_id, COUNT(*) AS open + FROM relay_assignment_migrations migration + WHERE migration.completed_at IS NULL AND migration.aborted_at IS NULL + GROUP BY migration.source_cell_id, migration.target_cell_id + ORDER BY migration.source_cell_id, migration.target_cell_id` + ) + const inactiveRows = await database.query( + `SELECT migration.source_cell_id, migration.target_cell_id, + COUNT(*) AS inactive, + COALESCE(SUM(CASE WHEN ${ABANDONED_REGISTERED_MIGRATION} + THEN 1 ELSE 0 END), 0) AS abandoned + FROM relay_assignment_migrations migration + WHERE migration.target_registered_at IS NOT NULL + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL + AND migration.expires_at <= ? + AND NOT EXISTS ( + SELECT 1 FROM relay_assignment_activity_leases target_control + WHERE target_control.user_id = migration.user_id + AND target_control.relay_host_id = migration.relay_host_id + AND target_control.cell_id = migration.target_cell_id + AND target_control.activity_kind = 'control' + AND target_control.activity_id NOT LIKE 'control-pending:%' + ) + GROUP BY migration.source_cell_id, migration.target_cell_id + ORDER BY migration.source_cell_id, migration.target_cell_id`, + [now, abandonedBefore, abandonedBefore, now] + ) + const inactiveByPair = new Map( + inactiveRows.map((row) => [ + JSON.stringify([text(row, 'source_cell_id'), text(row, 'target_cell_id')]), + row + ]) + ) + const pairs = openRows.map((row) => { + const sourceCellId = text(row, 'source_cell_id') + const targetCellId = text(row, 'target_cell_id') + const inactive = inactiveByPair.get(JSON.stringify([sourceCellId, targetCellId])) + return { + sourceCellId, + targetCellId, + open: integer(row, 'open'), + inactive: integer(inactive, 'inactive'), + abandoned: integer(inactive, 'abandoned') + } + }) + return { + open: pairs.reduce((total, pair) => total + pair.open, 0), + inactive: pairs.reduce((total, pair) => total + pair.inactive, 0), + abandoned: pairs.reduce((total, pair) => total + pair.abandoned, 0), + pairs + } +} + +export function formatRegisteredMigrationInventory( + inventory: RegisteredMigrationInventory +): string[] { + const lines = [ + `[orca-relay] migration inventory open=${inventory.open}` + + ` expiredRegisteredInactive=${inventory.inactive}` + + ` abandonedRegistered=${inventory.abandoned}` + ] + for (const pair of inventory.pairs) { + lines.push( + `[orca-relay] migration pair sourceCellId=${pair.sourceCellId}` + + ` targetCellId=${pair.targetCellId} open=${pair.open}` + + ` expiredRegisteredInactive=${pair.inactive}` + + ` abandoned=${pair.abandoned}` + ) + } + return lines +} + +function text(row: SqlRow | undefined, column: string): string { + return String(row?.[column] ?? '') +} + +function integer(row: SqlRow | undefined, column: string): number { + return Number(row?.[column] ?? 0) +} diff --git a/cloud/apps/relay/src/relay-background-operation.test.ts b/cloud/apps/relay/src/relay-background-operation.test.ts new file mode 100644 index 00000000000..2c85317ba67 --- /dev/null +++ b/cloud/apps/relay/src/relay-background-operation.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from 'vitest' +import { runRelayBackgroundOperation } from './relay-background-operation.js' + +describe('relay background operations', () => { + it('redacts free-form failure messages that could carry secrets', async () => { + const warn = vi.fn() + + await expect( + runRelayBackgroundOperation( + () => Promise.reject(new Error('postgresql://secret@database.invalid/relay')), + '[orca-relay] credential cleanup failed', + warn + ) + ).resolves.toBeUndefined() + + expect(warn).toHaveBeenCalledWith('[orca-relay] credential cleanup failed: Error: redacted') + expect(String(warn.mock.calls[0])).not.toContain('secret') + }) + + it('logs invariant slugs and SQLSTATE codes verbatim', async () => { + const warn = vi.fn() + const locked = Object.assign(new Error('regional_rehome_assignment_mismatch'), { + code: '55P03' + }) + + await runRelayBackgroundOperation( + () => Promise.reject(locked), + '[orca-relay] assignment cleanup failed', + warn + ) + + expect(warn).toHaveBeenCalledWith( + '[orca-relay] assignment cleanup failed: Error: regional_rehome_assignment_mismatch code=55P03' + ) + }) + + it('does not warn after successful maintenance work', async () => { + const warn = vi.fn() + + await runRelayBackgroundOperation(() => Promise.resolve(), 'unused', warn) + + expect(warn).not.toHaveBeenCalled() + }) +}) diff --git a/cloud/apps/relay/src/relay-background-operation.ts b/cloud/apps/relay/src/relay-background-operation.ts new file mode 100644 index 00000000000..be3943b7b1a --- /dev/null +++ b/cloud/apps/relay/src/relay-background-operation.ts @@ -0,0 +1,28 @@ +type Warn = (message: string) => void + +// A swallowed error hides which sweep step is failing (a silently dead +// cleanup chain stalled production rehoming for hours), but raw messages can +// embed secrets such as connection strings. Only two provably inert shapes +// are logged: this codebase's snake_case invariant slugs, and five-character +// SQLSTATE codes; everything else stays redacted. +function describeFailure(error: unknown): string { + const name = error instanceof Error ? error.name : typeof error + const message = error instanceof Error ? error.message : '' + const slug = /^[a-z0-9_]{1,64}$/.test(message) ? message : 'redacted' + const code = (error as { code?: unknown } | null)?.code + const sqlState = typeof code === 'string' && /^[0-9A-Z]{5}$/.test(code) ? ` code=${code}` : '' + return `${name}: ${slug}${sqlState}` +} + +export async function runRelayBackgroundOperation( + operation: () => Promise, + failureMessage: string, + warn: Warn = console.warn +): Promise { + try { + await operation() + } catch (error) { + // Dependency outages must fail readiness without crashing liveness. + warn(`${failureMessage}: ${describeFailure(error)}`) + } +} diff --git a/cloud/apps/relay/src/relay-connection-hard-cap.blackbox.test.ts b/cloud/apps/relay/src/relay-connection-hard-cap.blackbox.test.ts new file mode 100644 index 00000000000..230711e27d8 --- /dev/null +++ b/cloud/apps/relay/src/relay-connection-hard-cap.blackbox.test.ts @@ -0,0 +1,301 @@ +import { createHash, createHmac } from 'node:crypto' +import { generateKeyPair, exportJWK, SignJWT } from 'jose' +import { mkdtempSync, rmSync } from 'node:fs' +import { createServer, type Server } from 'node:http' +import { createServer as createNetServer } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + buildHostProofMacInput, + HOST_CHALLENGE_PLAINTEXT_DOMAIN +} from '@orca-cloud/relay-contract' +import nacl from 'tweetnacl' +import { afterEach, describe, expect, it, vi } from 'vitest' +import WebSocket from 'ws' +import type { RawData } from 'ws' +import type { RelayConfig } from './config.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' +import { createRelayServer } from './relay-server.js' + +async function unusedPort(): Promise { + const server = createNetServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('missing test port') + await new Promise((resolve) => server.close(() => resolve())) + return address.port +} + +function openSocket(url: string, headers: Record = {}): Promise { + return new Promise((resolve, reject) => { + const socket = new WebSocket(url, { headers }) + socket.once('open', () => resolve(socket)) + socket.once('error', reject) + }) +} + +function rejectedUpgrade( + url: string, + headers: Record = {} +): Promise { + return new Promise((resolve, reject) => { + const socket = new WebSocket(url, { headers }) + socket.once('open', () => reject(new Error(`upgrade unexpectedly opened: ${url}`))) + socket.once('unexpected-response', (_request, response) => { + response.resume() + resolve(response.statusCode) + }) + socket.once('error', () => undefined) + }) +} + +function nextMessage(socket: WebSocket): Promise> { + return new Promise((resolve, reject) => { + socket.once('message', (data: RawData) => { + try { + resolve(JSON.parse(data.toString()) as Record) + } catch (error) { + reject(error) + } + }) + socket.once('error', reject) + }) +} + +async function proveControl( + socket: WebSocket, + hostId: string, + keyPair: nacl.BoxKeyPair, + rebind?: { secret: string; generation: number } +): Promise> { + socket.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: hostId, + assignmentEpoch: 1, + hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'), + appVersion: 'test', + ...(rebind + ? { + controlResumeSecret: rebind.secret, + previousGeneration: rebind.generation + } + : {}) + }) + ) + const challenge = await nextMessage(socket) + const plaintext = nacl.box.open( + Buffer.from(String(challenge.ciphertextB64), 'base64'), + Buffer.from(String(challenge.nonceB64), 'base64'), + Buffer.from(String(challenge.relayEphemeralPublicKeyB64), 'base64'), + keyPair.secretKey + ) + if (!plaintext) throw new Error('host challenge did not decrypt') + const domainLength = new TextEncoder().encode( + `${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0` + ).length + const transcriptLength = new DataView( + plaintext.buffer, + plaintext.byteOffset + domainLength, + 4 + ).getUint32(0, false) + const transcriptStart = domainLength + 4 + const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength) + const secret = plaintext.slice(transcriptStart + transcriptLength) + socket.send( + JSON.stringify({ + type: 'host-challenge-ack', + challengeId: challenge.challengeId, + proofB64: createHmac('sha256', secret) + .update(buildHostProofMacInput(transcript)) + .digest('base64') + }) + ) + return await nextMessage(socket) +} + +describe('relay connection hard cap', () => { + const cleanup: Array<() => Promise | void> = [] + + afterEach(async () => { + for (const close of cleanup.splice(0).reverse()) await close() + }) + + it('preserves control headroom and never disturbs established sockets', async () => { + const keys = await generateKeyPair('ES256') + const publicJwk = await exportJWK(keys.publicKey) + const adminKeys = await generateKeyPair('RS256') + const adminPublicJwk = await exportJWK(adminKeys.publicKey) + const jwks = createServer((_request, response) => { + response.setHeader('content-type', 'application/json') + response.end( + JSON.stringify({ + keys: [ + { ...publicJwk, kid: 'test-key', alg: 'ES256', use: 'sig' }, + { ...adminPublicJwk, kid: 'admin-key', alg: 'RS256', use: 'sig' } + ] + }) + ) + }) + await new Promise((resolve) => jwks.listen(0, '127.0.0.1', resolve)) + cleanup.push(() => new Promise((resolve) => jwks.close(() => resolve()))) + const jwksAddress = jwks.address() + if (!jwksAddress || typeof jwksAddress === 'string') throw new Error('missing JWKS address') + const issuer = `http://127.0.0.1:${jwksAddress.port}` + const port = await unusedPort() + const relayUrl = `http://127.0.0.1:${port}` + const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-hard-cap-')) + cleanup.push(() => rmSync(dataDir, { recursive: true, force: true })) + const database: RelayDatabase = await openRelayDatabase({ dataDir }) + cleanup.push(() => database.close()) + const config = { + port, + publicUrl: relayUrl, + cellUrl: relayUrl, + authIssuer: issuer, + authAudience: 'orca-relay', + jwksUrl: `${issuer}/jwks`, + assignmentSigningKey: new TextEncoder().encode('test-assignment-key-with-at-least-32-bytes'), + role: 'combined', + cellId: 'combined', + cells: [ + { + id: 'combined', + url: relayUrl, + capacityRequests: 900, + connectionHardCap: 600, + connectionUnobservedBound: 60 + } + ], + adminAudience: `${relayUrl}/admin`, + deployServiceAccount: 'deploy@example.com', + runtimeServiceAccount: 'runtime@example.com', + connectionHardCap: 600, + connectionUnobservedBound: 60, + adminJwksUrl: `${issuer}/jwks`, + databasePoolMax: 10, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir + } satisfies RelayConfig + const relay = createRelayServer(config, database, { + connectionLedgerLimits: { hardCap: 5, controlReserve: 1 } + }) + relay.server.listen(port, '127.0.0.1') + await new Promise((resolve) => relay.server.once('listening', resolve)) + cleanup.push(() => new Promise((resolve) => relay.server.close(() => resolve()))) + const wsUrl = relayUrl.replace('http:', 'ws:') + const adminToken = await new SignJWT({ + email: 'deploy@example.com', + email_verified: true + }) + .setProtectedHeader({ alg: 'RS256', kid: 'admin-key' }) + .setIssuer('https://accounts.google.com') + .setAudience(`${relayUrl}/admin`) + .setSubject('deploy-subject') + .setIssuedAt() + .setExpirationTime('5m') + .sign(adminKeys.privateKey) + const statusResponse = await fetch(`${relayUrl}/v1/admin/runtime-status`, { + method: 'POST', + headers: { + authorization: `Bearer ${adminToken}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1 }) + }) + expect(statusResponse.status).toBe(200) + expect(await statusResponse.json()).toMatchObject({ + connectionCapacity: { + hardCap: 600, + controlRebindReserve: 100, + ordinaryConnectionLimit: 500, + unobservedBound: 60, + normalAdmissionPause: 440 + } + }) + + const unmatchedData = await Promise.all( + ['unmatched-1', 'unmatched-2', 'unmatched-3'].map((connectionId) => + openSocket(`${wsUrl}/v1/host/data/${connectionId}`) + ) + ) + cleanup.push(() => unmatchedData.forEach((socket) => socket.terminate())) + expect(relay.runtimeCounts().enforcedConnectionUnits).toBe(3) + expect(await rejectedUpgrade(`${wsUrl}/v1/connect/abcdefghijklmnop`)).toBe(503) + expect(unmatchedData.every((socket) => socket.readyState === WebSocket.OPEN)).toBe(true) + + const hostKeyPair = nacl.box.keyPair() + const hostId = createHash('sha256') + .update(hostKeyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const token = await new SignJWT({ + prof: 'profile-1', + org: 'org-1', + purpose: 'host-control', + relayHostId: hostId + }) + .setProtectedHeader({ alg: 'ES256', kid: 'test-key' }) + .setIssuer(issuer) + .setAudience('orca-relay') + .setSubject('user-1') + .setIssuedAt() + .setExpirationTime('5m') + .sign(keys.privateKey) + const controlHeaders = { authorization: `Bearer ${token}` } + const control = await openSocket(`${wsUrl}/v1/host/control`, controlHeaders) + cleanup.push(() => control.terminate()) + const controlAck = await proveControl(control, hostId, hostKeyPair) + expect(controlAck).toMatchObject({ type: 'host-hello-ack', generation: 1 }) + expect(relay.runtimeCounts().enforcedConnectionUnits).toBe(4) + expect(await rejectedUpgrade(`${wsUrl}/v1/host/data/another`)).toBe(503) + const unrelatedToken = await new SignJWT({ + prof: 'profile-1', + purpose: 'host-control', + relayHostId: 'ponmlkjihgfedcba' + }) + .setProtectedHeader({ alg: 'ES256', kid: 'test-key' }) + .setIssuer(issuer) + .setAudience('orca-relay') + .setSubject('user-2') + .setIssuedAt() + .setExpirationTime('5m') + .sign(keys.privateKey) + expect( + await rejectedUpgrade(`${wsUrl}/v1/host/control`, { + authorization: `Bearer ${unrelatedToken}` + }) + ).toBe(503) + const failedBorrower = await openSocket(`${wsUrl}/v1/host/control`, controlHeaders) + cleanup.push(() => failedBorrower.terminate()) + expect(relay.runtimeCounts().enforcedConnectionUnits).toBe(5) + const failedBorrowerClosed = new Promise((resolve) => + failedBorrower.once('close', (code) => resolve(code)) + ) + failedBorrower.send(JSON.stringify({ type: 'host-hello' })) + expect(await failedBorrowerClosed).toBe(4401) + await vi.waitFor(() => expect(relay.runtimeCounts().enforcedConnectionUnits).toBe(4)) + expect(control.readyState).toBe(WebSocket.OPEN) + const replacementControl = await openSocket(`${wsUrl}/v1/host/control`, controlHeaders) + cleanup.push(() => replacementControl.terminate()) + expect(relay.runtimeCounts().enforcedConnectionUnits).toBe(5) + const controlClosed = new Promise((resolve) => + control.once('close', (code) => resolve(code)) + ) + const replacementAck = await proveControl(replacementControl, hostId, hostKeyPair, { + secret: String(controlAck.controlResumeSecret), + generation: 1 + }) + expect(replacementAck).toMatchObject({ type: 'host-hello-ack', generation: 1 }) + expect(await controlClosed).toBe(4408) + await vi.waitFor(() => expect(relay.runtimeCounts().enforcedConnectionUnits).toBe(4)) + expect(relay.runtimeCounts().enforcedConnectionUnits).toBe(4) + }) +}) diff --git a/cloud/apps/relay/src/relay-connection-ledger.test.ts b/cloud/apps/relay/src/relay-connection-ledger.test.ts new file mode 100644 index 00000000000..f807c34d20d --- /dev/null +++ b/cloud/apps/relay/src/relay-connection-ledger.test.ts @@ -0,0 +1,172 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it } from 'vitest' +import type WebSocket from 'ws' +import { RelayConnectionLedger } from './relay-connection-ledger.js' + +function socket(): { webSocket: WebSocket; close: () => void } { + const emitter = new EventEmitter() + return { + webSocket: emitter as WebSocket, + close: () => emitter.emit('close') + } +} + +describe('RelayConnectionLedger', () => { + it('orders connection admissions before covering absolute snapshots', () => { + const ledger = new RelayConnectionLedger(6, 2) + const upgrade = ledger.tryReserveControl(false)! + const beforePromotion = ledger.snapshot() + const controlSocket = socket() + upgrade.promote(controlSocket.webSocket) + const afterPromotion = ledger.snapshot() + + expect(beforePromotion.inclusionWatermark).toBeGreaterThanOrEqual( + upgrade.inclusionWatermark + ) + expect(afterPromotion.inclusionWatermark).toBeGreaterThan( + beforePromotion.inclusionWatermark + ) + expect(afterPromotion).toMatchObject({ + physicalConnections: 1, + inFlightConnections: 0, + enforcedConnectionUnits: 1 + }) + }) + + it.each([600, 1_000, 3_000])( + 'enforces the %i-unit boundary with 100 control units reserved', + (hardCap) => { + const ledger = new RelayConnectionLedger(hardCap, 100) + const sockets: Array<{ webSocket: WebSocket; close: () => void }> = [] + for (let index = 0; index < (hardCap - 100) / 2; index++) { + const admission = ledger.tryReservePhone() + expect(admission).not.toBeNull() + const phoneSocket = socket() + admission!.upgrade.promote(phoneSocket.webSocket) + admission!.hostData.bind(`connection-${index}`) + sockets.push(phoneSocket) + } + expect(ledger.tryReservePhone()).toBeNull() + expect(ledger.tryReserveControl(false)).toBeNull() + for (let index = 0; index < 100; index++) { + const upgrade = ledger.tryReserveControl(true) + expect(upgrade).not.toBeNull() + const controlSocket = socket() + upgrade!.promote(controlSocket.webSocket) + sockets.push(controlSocket) + } + + expect(ledger.counts().enforcedConnectionUnits).toBe(hardCap) + expect(ledger.tryReserveControl(true)).toBeNull() + sockets.at(-1)!.close() + const replacement = ledger.tryReserveControl(true) + expect(replacement).not.toBeNull() + replacement!.release() + } + ) + + it('reserves both phone legs below the control reserve', () => { + const ledger = new RelayConnectionLedger(6, 2) + const first = ledger.tryReservePhone() + const second = ledger.tryReservePhone() + + expect(first).not.toBeNull() + expect(second).not.toBeNull() + expect(ledger.tryReservePhone()).toBeNull() + expect(ledger.counts()).toEqual({ + physicalConnections: 0, + inFlightConnections: 2, + reservedConnectionUnits: 2, + enforcedConnectionUnits: 4 + }) + expect(ledger.tryReserveControl(false)).toBeNull() + expect(ledger.tryReserveControl(true)).not.toBeNull() + expect(ledger.tryReserveControl(true)).not.toBeNull() + expect(ledger.tryReserveControl(true)).toBeNull() + }) + + it('transfers a pending host-data unit without increasing enforced capacity', () => { + const ledger = new RelayConnectionLedger(6, 2) + const phone = ledger.tryReservePhone()! + const phoneSocket = socket() + phone.upgrade.promote(phoneSocket.webSocket) + phone.hostData.bind('connection-1') + + const data = ledger.tryReserveHostData('connection-1')! + expect(ledger.counts().enforcedConnectionUnits).toBe(2) + const dataSocket = socket() + data.promote(dataSocket.webSocket) + expect(ledger.counts().enforcedConnectionUnits).toBe(2) + expect(data.commitHostData()).toBe(true) + + dataSocket.close() + dataSocket.close() + expect(ledger.counts()).toEqual({ + physicalConnections: 1, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 1 + }) + }) + + it('restores a claimed reservation after rejected host-data authentication', () => { + const ledger = new RelayConnectionLedger(6, 2) + const phone = ledger.tryReservePhone()! + const phoneSocket = socket() + phone.upgrade.promote(phoneSocket.webSocket) + phone.hostData.bind('connection-1') + + const rejected = ledger.tryReserveHostData('connection-1')! + const rejectedSocket = socket() + rejected.promote(rejectedSocket.webSocket) + rejectedSocket.close() + + expect(ledger.counts()).toEqual({ + physicalConnections: 1, + inFlightConnections: 0, + reservedConnectionUnits: 1, + enforcedConnectionUnits: 2 + }) + expect(ledger.tryReserveHostData('connection-1')).not.toBeNull() + }) + + it('does not restore a claimed reservation after the phone closes', () => { + const ledger = new RelayConnectionLedger(6, 2) + const phone = ledger.tryReservePhone()! + const phoneSocket = socket() + phone.upgrade.promote(phoneSocket.webSocket) + phone.hostData.bind('connection-1') + const data = ledger.tryReserveHostData('connection-1')! + const dataSocket = socket() + data.promote(dataSocket.webSocket) + + phone.hostData.release() + phoneSocket.close() + dataSocket.close() + + expect(ledger.counts()).toEqual({ + physicalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }) + expect(ledger.tryReserveHostData('connection-1')).not.toBeNull() + }) + + it('releases failed in-flight upgrades exactly once', () => { + const ledger = new RelayConnectionLedger(6, 2) + const phone = ledger.tryReservePhone()! + + phone.upgrade.release() + phone.upgrade.release() + phone.hostData.release() + phone.hostData.release() + + expect(ledger.counts()).toEqual({ + physicalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }) + }) +}) diff --git a/cloud/apps/relay/src/relay-connection-ledger.ts b/cloud/apps/relay/src/relay-connection-ledger.ts new file mode 100644 index 00000000000..9c678695fa8 --- /dev/null +++ b/cloud/apps/relay/src/relay-connection-ledger.ts @@ -0,0 +1,229 @@ +import type WebSocket from 'ws' + +export type RelayConnectionLedgerCounts = { + physicalConnections: number + inFlightConnections: number + reservedConnectionUnits: number + enforcedConnectionUnits: number +} + +export type RelayConnectionLedgerSnapshot = RelayConnectionLedgerCounts & { + inclusionWatermark: number +} + +export type PendingHostDataReservation = { + bind: (connectionId: string) => void + release: () => void +} + +type ReservationState = 'reserved' | 'claimed' | 'consumed' | 'released' +type UpgradeState = 'in-flight' | 'physical' | 'released' + +class HostDataReservation implements PendingHostDataReservation { + private state: ReservationState = 'reserved' + private connectionId: string | null = null + + constructor(private readonly ledger: RelayConnectionLedger) {} + + bind(connectionId: string): void { + if (this.state !== 'reserved' || this.connectionId !== null) { + throw new Error('host_data_reservation_already_bound') + } + this.connectionId = connectionId + this.ledger.bindHostData(connectionId, this) + } + + release(): void { + if (this.state === 'reserved') { + this.ledger.releaseReserved(this) + } + if (this.state !== 'consumed') { + this.state = 'released' + } + } + + claim(): void { + if (this.state !== 'reserved') throw new Error('host_data_reservation_unavailable') + this.state = 'claimed' + } + + consume(): boolean { + if (this.state !== 'claimed') return false + this.state = 'consumed' + return true + } + + restore(): void { + if (this.state !== 'claimed') return + this.state = this.ledger.restoreReserved(this) ? 'reserved' : 'released' + } + + matches(connectionId: string): boolean { + return this.connectionId === connectionId + } + + get boundConnectionId(): string | null { + return this.connectionId + } +} + +export class RelayConnectionUpgrade { + private state: UpgradeState = 'in-flight' + + constructor( + private readonly ledger: RelayConnectionLedger, + readonly inclusionWatermark: number, + private readonly hostDataReservation: HostDataReservation | null = null + ) {} + + promote(socket: WebSocket): void { + if (this.state !== 'in-flight') throw new Error('connection_upgrade_not_in_flight') + this.state = 'physical' + this.ledger.promoteUpgrade() + socket.once('close', () => this.release()) + } + + commitHostData(): boolean { + return this.hostDataReservation?.consume() ?? false + } + + release(): void { + if (this.state === 'released') return + if (this.state === 'in-flight') this.ledger.releaseUpgrade() + else this.ledger.releasePhysical() + this.state = 'released' + this.hostDataReservation?.restore() + } +} + +export type PhoneConnectionAdmission = { + upgrade: RelayConnectionUpgrade + hostData: PendingHostDataReservation +} + +export class RelayConnectionLedger { + private physicalConnections = 0 + private inFlightConnections = 0 + private reservedConnectionUnits = 0 + private inclusionWatermark = 0 + private readonly hostDataByConnectionId = new Map() + + constructor( + private readonly hardCap: number, + private readonly controlReserve: number + ) { + if (hardCap <= 0 || controlReserve < 0 || controlReserve >= hardCap) { + throw new Error('invalid_connection_ledger_capacity') + } + } + + tryReserveControl(rebind: boolean): RelayConnectionUpgrade | null { + return this.reserveUpgrade(rebind ? this.hardCap : this.normalAdmissionLimit) + } + + tryReservePhone(): PhoneConnectionAdmission | null { + if (this.enforcedConnectionUnits + 2 > this.normalAdmissionLimit) return null + const reservation = new HostDataReservation(this) + const inclusionWatermark = this.advanceWatermark() + this.inFlightConnections++ + this.reservedConnectionUnits++ + return { + upgrade: new RelayConnectionUpgrade(this, inclusionWatermark), + hostData: reservation + } + } + + tryReserveHostData(connectionId: string): RelayConnectionUpgrade | null { + const reservation = this.hostDataByConnectionId.get(connectionId) + if (reservation?.matches(connectionId)) { + this.hostDataByConnectionId.delete(connectionId) + reservation.claim() + const inclusionWatermark = this.advanceWatermark() + this.reservedConnectionUnits-- + this.inFlightConnections++ + return new RelayConnectionUpgrade(this, inclusionWatermark, reservation) + } + return this.reserveUpgrade(this.normalAdmissionLimit) + } + + counts(): RelayConnectionLedgerCounts { + return { + physicalConnections: this.physicalConnections, + inFlightConnections: this.inFlightConnections, + reservedConnectionUnits: this.reservedConnectionUnits, + enforcedConnectionUnits: this.enforcedConnectionUnits + } + } + + snapshot(): RelayConnectionLedgerSnapshot { + return { + ...this.counts(), + inclusionWatermark: this.advanceWatermark() + } + } + + bindHostData(connectionId: string, reservation: HostDataReservation): void { + if (this.hostDataByConnectionId.has(connectionId)) { + throw new Error('host_data_reservation_conflict') + } + this.hostDataByConnectionId.set(connectionId, reservation) + } + + releaseReserved(reservation: HostDataReservation): void { + const connectionId = reservation.boundConnectionId + if (connectionId && this.hostDataByConnectionId.get(connectionId) === reservation) { + this.hostDataByConnectionId.delete(connectionId) + } + this.advanceWatermark() + this.reservedConnectionUnits-- + } + + restoreReserved(reservation: HostDataReservation): boolean { + const connectionId = reservation.boundConnectionId + if (!connectionId || this.hostDataByConnectionId.has(connectionId)) { + return false + } + this.advanceWatermark() + this.reservedConnectionUnits++ + this.hostDataByConnectionId.set(connectionId, reservation) + return true + } + + promoteUpgrade(): void { + this.advanceWatermark() + this.inFlightConnections-- + this.physicalConnections++ + } + + releaseUpgrade(): void { + this.advanceWatermark() + this.inFlightConnections-- + } + + releasePhysical(): void { + this.advanceWatermark() + this.physicalConnections-- + } + + private reserveUpgrade(limit: number): RelayConnectionUpgrade | null { + if (this.enforcedConnectionUnits + 1 > limit) return null + const inclusionWatermark = this.advanceWatermark() + this.inFlightConnections++ + return new RelayConnectionUpgrade(this, inclusionWatermark) + } + + private advanceWatermark(): number { + this.inclusionWatermark++ + return this.inclusionWatermark + } + + private get normalAdmissionLimit(): number { + return this.hardCap - this.controlReserve + } + + private get enforcedConnectionUnits(): number { + return ( + this.physicalConnections + this.inFlightConnections + this.reservedConnectionUnits + ) + } +} diff --git a/cloud/apps/relay/src/relay-first-frame-failure.blackbox.test.ts b/cloud/apps/relay/src/relay-first-frame-failure.blackbox.test.ts new file mode 100644 index 00000000000..57e47a065d3 --- /dev/null +++ b/cloud/apps/relay/src/relay-first-frame-failure.blackbox.test.ts @@ -0,0 +1,124 @@ +import { createServer as createNetServer } from 'node:net' +import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract' +import { afterEach, describe, expect, it, vi } from 'vitest' +import WebSocket from 'ws' +import type { RelayConfig } from './config.js' +import type { RelayDatabase } from './database.js' +import { createRelayServer } from './relay-server.js' + +async function unusedPort(): Promise { + const server = createNetServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('missing test port') + await new Promise((resolve) => server.close(() => resolve())) + return address.port +} + +function openSocket(url: string): Promise { + return new Promise((resolve, reject) => { + const socket = new WebSocket(url) + socket.once('open', () => resolve(socket)) + socket.once('error', reject) + }) +} + +describe('relay first-frame failures', () => { + const cleanup: Array<() => Promise | void> = [] + + afterEach(async () => { + for (const close of cleanup.splice(0).reverse()) await close() + vi.restoreAllMocks() + }) + + it('contains phone authentication failures and releases connection capacity', async () => { + const port = await unusedPort() + const relayUrl = `http://127.0.0.1:${port}` + const poolTimeout = new Error('injected pool connection timeout') + const database: RelayDatabase = { + query: vi.fn(async () => { + throw poolTimeout + }), + queryLocked: vi.fn(async () => { + throw poolTimeout + }), + transaction: vi.fn(async (operation) => await operation(database)), + close: vi.fn(async () => undefined) + } + const config = { + port, + publicUrl: relayUrl, + cellUrl: relayUrl, + authIssuer: 'https://auth.example.com', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.com/jwks', + assignmentSigningKey: new Uint8Array(32), + role: 'cell', + cellId: 'production-gce-c3', + cells: [{ id: 'production-gce-c3', url: relayUrl, capacityRequests: 4_000 }], + adminAudience: `${relayUrl}/admin`, + deployServiceAccount: 'deploy@example.com', + runtimeServiceAccount: 'runtime@example.com', + connectionHardCap: 600, + connectionUnobservedBound: 60, + adminJwksUrl: 'https://auth.example.com/admin-jwks', + databasePoolMax: 10, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './test-data' + } satisfies RelayConfig + const relay = createRelayServer(config, database, { + connectionLedgerLimits: { hardCap: 5, controlReserve: 1 } + }) + relay.server.listen(port, '127.0.0.1') + await new Promise((resolve) => relay.server.once('listening', resolve)) + cleanup.push(() => new Promise((resolve) => relay.server.close(() => resolve()))) + vi.spyOn(console, 'log').mockImplementation(() => undefined) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const unhandled: unknown[] = [] + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason) + } + process.on('unhandledRejection', onUnhandled) + cleanup.push(() => { + process.off('unhandledRejection', onUnhandled) + }) + + const socket = await openSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop` + ) + cleanup.push(() => socket.terminate()) + expect(relay.connectionSnapshot()).toMatchObject({ + physicalConnections: 1, + reservedConnectionUnits: 1, + enforcedConnectionUnits: 2 + }) + const closed = new Promise((resolve) => + socket.once('close', (code) => resolve(code)) + ) + socket.send( + JSON.stringify({ + type: 'relay-auth', + v: 1, + mode: 'connect', + credential: Buffer.alloc(32, 1).toString('base64url') + }) + ) + + expect(await closed).toBe(RELAY_CLOSE_CODE.LIMIT_EXCEEDED) + await vi.waitFor(() => + expect(relay.connectionSnapshot()).toMatchObject({ + physicalConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }) + ) + await new Promise((resolve) => setImmediate(resolve)) + expect(unhandled).toEqual([]) + }) +}) diff --git a/cloud/apps/relay/src/relay-host-log-digest.ts b/cloud/apps/relay/src/relay-host-log-digest.ts new file mode 100644 index 00000000000..b232170a6d4 --- /dev/null +++ b/cloud/apps/relay/src/relay-host-log-digest.ts @@ -0,0 +1,7 @@ +import { createHash } from 'node:crypto' + +// Store-level 503s were previously silent; the digest keeps per-host log +// correlation possible without emitting the raw relay host id. +export function relayHostLogDigest(relayHostId: string): string { + return createHash('sha256').update(relayHostId).digest('hex').slice(0, 12) +} diff --git a/cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts b/cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts new file mode 100644 index 00000000000..924523685e3 --- /dev/null +++ b/cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts @@ -0,0 +1,153 @@ +import { createHash } from 'node:crypto' +import { createServer, type Server } from 'node:http' +import { createServer as createNetServer } from 'node:net' +import { exportJWK, generateKeyPair, SignJWT } from 'jose' +import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract' +import nacl from 'tweetnacl' +import { afterEach, describe, expect, it, vi } from 'vitest' +import WebSocket from 'ws' +import type { RelayConfig } from './config.js' +import type { RelayDatabase } from './database.js' +import { createRelayServer } from './relay-server.js' + +async function unusedPort(): Promise { + const server = createNetServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('missing test port') + await new Promise((resolve) => server.close(() => resolve())) + return address.port +} + +describe('relay host proof failures', () => { + const cleanup: Array<() => Promise | void> = [] + + afterEach(async () => { + for (const close of cleanup.splice(0).reverse()) await close() + vi.restoreAllMocks() + }) + + // The production crash signature: a pg-pool connect timeout rejecting out of + // verifyCellAssignment inside beginProof killed whole cells as an unhandled + // rejection. The guard must contain it to this one handshake. + it('contains a database timeout during host hello to one socket', async () => { + const keys = await generateKeyPair('ES256') + const publicJwk = await exportJWK(keys.publicKey) + const jwksServer: Server = createServer((_request, response) => { + response.setHeader('content-type', 'application/json') + response.end( + JSON.stringify({ keys: [{ ...publicJwk, kid: 'test-key', alg: 'ES256', use: 'sig' }] }) + ) + }) + await new Promise((resolve) => jwksServer.listen(0, '127.0.0.1', resolve)) + cleanup.push(() => new Promise((resolve) => jwksServer.close(() => resolve()))) + const jwksAddress = jwksServer.address() + if (!jwksAddress || typeof jwksAddress === 'string') throw new Error('missing JWKS address') + const issuer = `http://127.0.0.1:${jwksAddress.port}` + + const port = await unusedPort() + const relayUrl = `http://127.0.0.1:${port}` + const poolTimeout = new Error('Connection terminated due to connection timeout') + const database: RelayDatabase = { + query: vi.fn(async () => { + throw poolTimeout + }), + queryLocked: vi.fn(async () => { + throw poolTimeout + }), + transaction: vi.fn(async (operation) => await operation(database)), + close: vi.fn(async () => undefined) + } + const config = { + port, + publicUrl: relayUrl, + cellUrl: relayUrl, + authIssuer: issuer, + authAudience: 'orca-relay', + jwksUrl: issuer, + assignmentSigningKey: new Uint8Array(32), + role: 'cell', + cellId: 'production-gce-c3', + cells: [{ id: 'production-gce-c3', url: relayUrl, capacityRequests: 4_000 }], + adminAudience: `${relayUrl}/admin`, + deployServiceAccount: 'deploy@example.com', + runtimeServiceAccount: 'runtime@example.com', + connectionHardCap: 600, + connectionUnobservedBound: 60, + adminJwksUrl: `${issuer}/admin-jwks`, + databasePoolMax: 10, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './test-data' + } satisfies RelayConfig + const relay = createRelayServer(config, database) + relay.server.listen(port, '127.0.0.1') + await new Promise((resolve) => relay.server.once('listening', resolve)) + cleanup.push(() => new Promise((resolve) => relay.server.close(() => resolve()))) + vi.spyOn(console, 'log').mockImplementation(() => undefined) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const unhandled: unknown[] = [] + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason) + } + process.on('unhandledRejection', onUnhandled) + cleanup.push(() => { + process.off('unhandledRejection', onUnhandled) + }) + + const keyPair = nacl.box.keyPair() + const hostId = createHash('sha256') + .update(keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const token = await new SignJWT({ + prof: 'profile-1', + org: 'org-1', + purpose: 'host-control', + relayHostId: hostId + }) + .setProtectedHeader({ alg: 'ES256', kid: 'test-key' }) + .setIssuer(issuer) + .setAudience('orca-relay') + .setSubject('user-1') + .setIssuedAt() + .setExpirationTime('5m') + .sign(keys.privateKey) + const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, { + headers: { authorization: `Bearer ${token}` }, + perMessageDeflate: false + }) + cleanup.push(() => socket.terminate()) + await new Promise((resolve, reject) => { + socket.once('open', resolve) + socket.once('error', reject) + }) + const closed = new Promise((resolve) => + socket.once('close', (code) => resolve(code)) + ) + socket.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: hostId, + assignmentEpoch: 1, + hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'), + appVersion: 'test' + }) + ) + + expect(await closed).toBe(RELAY_CLOSE_CODE.LIMIT_EXCEEDED) + expect( + warn.mock.calls.map((call) => String(call[0])) + ).toContain( + '[orca-relay] host hello proof failed: Connection terminated due to connection timeout' + ) + await new Promise((resolve) => setImmediate(resolve)) + expect(unhandled).toEqual([]) + }) +}) diff --git a/cloud/apps/relay/src/relay-observability.test.ts b/cloud/apps/relay/src/relay-observability.test.ts new file mode 100644 index 00000000000..2b9ceb0b72a --- /dev/null +++ b/cloud/apps/relay/src/relay-observability.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RelayDatabase } from './database.js' +import { observeRelayDatabase } from './observed-relay-database.js' +import { + observedRelayRequests, + RelayObservability, + type RelayProcessCounts +} from './relay-observability.js' + +const counts: RelayProcessCounts = { + totalConnections: 9, + preAuthConnections: 1, + controls: 2, + splices: 3, + pendingSplices: 1, + queuedBytes: 4096, + databasePoolTotal: 3, + databasePoolIdle: 0, + databasePoolWaiting: 2, + databasePoolWaitersMax: 3, + databasePoolOldestWaitMs: 750, + databasePoolWaitMsMax: 1_250 +} + +describe('relay observability', () => { + it('emits safe readiness dependency outcomes', () => { + const entries: Array> = [] + const observability = new RelayObservability( + { role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' }, + (entry) => entries.push(entry) + ) + + observability.recordReadiness({ + ready: false, + failure: 'sql_failed', + jwksLatencyMs: 12, + sqlLatencyMs: 2_001, + totalLatencyMs: 2_013 + }) + + expect(entries).toEqual([ + { + severity: 'WARNING', + message: 'Orca Relay readiness check', + event: 'orca_relay_readiness_check', + metricVersion: 1, + role: 'cell', + cellId: 'production-gce-c28', + region: 'asia-east2', + ready: false, + failure: 'sql_failed', + jwksLatencyMs: 12, + sqlLatencyMs: 2_001, + totalLatencyMs: 2_013 + } + ]) + }) + + it('excludes sockets stuck in closing state from observed relay work', () => { + expect(observedRelayRequests(counts)).toBe(7) + }) + + it('keeps rejection reasons separate per lane and resets them each flush', () => { + const entries: Array> = [] + const observability = new RelayObservability( + { role: 'director', cellId: 'director', region: 'us-central1' }, + (entry) => entries.push(entry) + ) + observability.recordAssignmentAdmission('placement-rejected') + observability.recordAssignmentRejectionReason('placement', 'host-rate-limited') + observability.recordAssignmentRejectionReason('placement', 'host-rate-limited') + observability.recordAssignmentRejectionReason('placement', 'queue-full') + observability.recordAssignmentRejectionReason('sticky', 'wait-timeout') + observability.flush(counts) + observability.flush(counts) + + expect(entries[0]).toMatchObject({ + placementAssignmentRejectionsDelta: 1, + placementRejectionsByReasonDelta: { 'host-rate-limited': 2, 'queue-full': 1 }, + stickyRejectionsByReasonDelta: { 'wait-timeout': 1 } + }) + expect(entries[1]).toMatchObject({ + placementRejectionsByReasonDelta: {}, + stickyRejectionsByReasonDelta: {} + }) + }) + + it('aggregates coarse region requests, selections, fallbacks, and outages', () => { + const entries: Array> = [] + const observability = new RelayObservability( + { role: 'director', cellId: 'director', region: 'us-central1' }, + (entry) => entries.push(entry) + ) + observability.recordRegionRequest('asia-east2') + observability.recordRegionRequest(undefined) + observability.recordRegionSelection({ + targetRegion: 'asia-east2', + selectedRegion: 'us-central1', + fallback: true + }) + observability.recordRegionSelection({ targetRegion: 'asia-east2', fallback: false }) + observability.flush(counts) + observability.flush(counts) + + expect(entries[0]).toMatchObject({ + requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 }, + selectedRegionsDelta: { 'us-central1': 1 }, + regionFallbacksDelta: { 'asia-east2': 1 }, + unavailableRegionsDelta: { 'asia-east2': 1 } + }) + expect(entries[1]).toMatchObject({ + requestedRegionsDelta: {}, + selectedRegionsDelta: {}, + regionFallbacksDelta: {}, + unavailableRegionsDelta: {} + }) + }) + + it('emits bounded aggregate runtime signals without identities or credentials', () => { + const entries: Array> = [] + const observability = new RelayObservability( + { role: 'cell', cellId: 'staging-c1', region: 'asia-east2' }, + (entry) => entries.push(entry) + ) + observability.recordAuth(true) + observability.recordAuth(false) + observability.recordForwardedBytes(123) + observability.recordHttp(45.6789) + observability.recordReconnect() + observability.recordSql(12.3456, true) + observability.recordSql(4, false) + observability.recordControlRenewal(2, 'renewed') + observability.recordControlRenewal(8, 'control_activity_not_found') + observability.recordControlRenewal(4, 'renewed') + observability.recordControlActivityRecovery(true) + observability.recordControlActivityRecovery(false) + observability.flush(counts) + observability.flush(counts) + + expect(entries[0]).toMatchObject({ + event: 'orca_relay_runtime_metrics', + metricVersion: 2, + role: 'cell', + cellId: 'staging-c1', + region: 'asia-east2', + ...counts, + forwardedBytesDelta: 123, + authSuccessesDelta: 1, + authFailuresDelta: 1, + reconnectsDelta: 1, + sqlQueriesDelta: 2, + sqlFailuresDelta: 1, + sqlLatencyMsMax: 12.346, + controlRenewalsByOutcomeDelta: { renewed: 2, control_activity_not_found: 1 }, + controlRenewalsDelta: 3, + controlRenewalSuccessesDelta: 2, + controlRenewalLeaseMissesDelta: 1, + controlRenewalLatencyMsP50: 4, + controlRenewalLatencyMsP95: 8, + controlRenewalLatencyMsMax: 8, + controlActivityRecoveriesDelta: 1, + controlActivityRecoveryFailuresDelta: 1, + httpLatencyMsMax: 45.679 + }) + expect(entries[1]).toMatchObject({ + forwardedBytesDelta: 0, + authSuccessesDelta: 0, + authFailuresDelta: 0, + reconnectsDelta: 0, + sqlQueriesDelta: 0, + sqlFailuresDelta: 0, + sqlLatencyMsMax: 0, + controlRenewalsByOutcomeDelta: {}, + controlRenewalsDelta: 0, + controlRenewalSuccessesDelta: 0, + controlRenewalLeaseMissesDelta: 0, + controlRenewalLatencyMsP50: 0, + controlRenewalLatencyMsP95: 0, + controlRenewalLatencyMsMax: 0, + controlActivityRecoveriesDelta: 0, + controlActivityRecoveryFailuresDelta: 0, + httpLatencyMsMax: 0 + }) + expect(JSON.stringify(entries)).not.toMatch(/token|credential|userId|relayHostId/) + }) + + it('aggregates control and splice closes as bounded per-reason deltas', () => { + const entries: Array> = [] + const observability = new RelayObservability( + { role: 'cell', cellId: 'staging-c1', region: 'us-central1' }, + (entry) => entries.push(entry) + ) + observability.recordControlClose(1006) + observability.recordControlClose(1006) + observability.recordControlClose(4402) + observability.recordSpliceClose('host-oversize-frame') + observability.recordSpliceClose('queue-limit') + observability.flush(counts) + observability.flush(counts) + + expect(entries[0]).toMatchObject({ + controlClosesByCodeDelta: { 1006: 2, 4402: 1 }, + spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 } + }) + expect(entries[1]).toMatchObject({ + controlClosesByCodeDelta: {}, + spliceClosesByTriggerDelta: {} + }) + }) + + it('observes successful and failed database calls including transactions', async () => { + const recordSql = vi.fn() + const underlying: RelayDatabase = { + query: vi.fn(async () => [{ ok: true }]), + queryLocked: vi.fn(async (sql, _params, options) => { + throw new Error( + options?.failIfUnavailable && sql === 'SELECT 3' + ? 'database_lock_unavailable' + : 'database unavailable' + ) + }), + transaction: async (operation) => await operation(underlying), + close: vi.fn(async () => {}) + } + const database = observeRelayDatabase(underlying, { + recordAuth: vi.fn(), + recordForwardedBytes: vi.fn(), + recordHttp: vi.fn(), + recordReconnect: vi.fn(), + recordSql + }) + + await database.query('SELECT 1') + await expect(database.transaction(async (tx) => await tx.queryLocked('SELECT 2'))).rejects.toThrow( + 'database unavailable' + ) + await expect( + database.queryLocked('SELECT 3', [], { failIfUnavailable: true }) + ).rejects.toThrow('database_lock_unavailable') + await expect( + database.queryLocked('SELECT 4', [], { failIfUnavailable: true }) + ).rejects.toThrow('database unavailable') + expect(recordSql).toHaveBeenCalledTimes(4) + expect(recordSql.mock.calls.map((call) => call[1])).toEqual([true, false, true, false]) + }) +}) diff --git a/cloud/apps/relay/src/relay-observability.ts b/cloud/apps/relay/src/relay-observability.ts new file mode 100644 index 00000000000..6125ede8d1a --- /dev/null +++ b/cloud/apps/relay/src/relay-observability.ts @@ -0,0 +1,336 @@ +import { monitorEventLoopDelay, performance } from 'node:perf_hooks' +import type { RelayRegion } from '@orca-cloud/relay-contract' +import type { ControlRenewalOutcome } from './assignment-store.js' +import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js' +import type { RelayReadinessObservation } from './relay-readiness.js' + +export type RelayRuntimeCounts = { + totalConnections: number + inFlightConnections?: number + reservedConnectionUnits?: number + enforcedConnectionUnits?: number + preAuthConnections: number + controls: number + splices: number + pendingSplices: number + queuedBytes: number +} + +export function observedRelayRequests(counts: RelayRuntimeCounts): number { + return counts.preAuthConnections + counts.controls + counts.splices + counts.pendingSplices +} + +export type RelayProcessCounts = RelayRuntimeCounts & PostgresPoolPressureCounts + +export type RegionalRehomeRuntimeSafety = { + observedAt: number + sqlFailures: number + reconnects: number + controlActivityRecoveryFailures: number +} + +export type RegionalRehomeSafetySnapshot = RegionalRehomeRuntimeSafety & { + databasePoolWaiting: number + databasePoolWaitersMax: number + databasePoolWaitMsMax: number +} + +export type AssignmentAdmissionOutcome = + | 'sticky' + | 'sticky-rejected' + | 'placement' + | 'placement-rejected' + +export type AssignmentAdmissionLane = 'sticky' | 'placement' + +export interface RelayRuntimeObserver { + recordAuth(success: boolean): void + recordForwardedBytes(bytes: number): void + recordHttp(durationMs: number): void + recordReconnect(): void + recordSql(durationMs: number, success: boolean): void + recordControlRenewal?(durationMs: number, outcome: ControlRenewalOutcome): void + recordControlActivityRecovery?(success: boolean): void + recordAssignmentAdmission?(outcome: AssignmentAdmissionOutcome): void + recordAssignmentRejectionReason?(lane: AssignmentAdmissionLane, reason: string): void + recordRegionRequest?(region: RelayRegion | undefined): void + recordRegionSelection?(input: { + targetRegion: RelayRegion + selectedRegion?: RelayRegion + fallback: boolean + }): void + recordControlClose?(code: number): void + recordSpliceClose?(trigger: string): void +} + +type RelayMetricDeltas = { + forwardedBytes: number + authSuccesses: number + authFailures: number + reconnects: number + sqlQueries: number + sqlFailures: number + sqlLatencyMsMax: number + httpLatencyMsMax: number + stickyAssignments: number + stickyAssignmentRejections: number + placementAssignments: number + placementAssignmentRejections: number + stickyRejectionsByReason: Record + placementRejectionsByReason: Record + requestedRegions: Record + selectedRegions: Record + regionFallbacks: Record + unavailableRegions: Record + controlClosesByCode: Record + spliceClosesByTrigger: Record + controlRenewalLatenciesMs: number[] + controlRenewalsByOutcome: Record + controlActivityRecoveries: number + controlActivityRecoveryFailures: number +} + +type MetricWriter = (entry: Record) => void + +const emptyDeltas = (): RelayMetricDeltas => ({ + forwardedBytes: 0, + authSuccesses: 0, + authFailures: 0, + reconnects: 0, + sqlQueries: 0, + sqlFailures: 0, + sqlLatencyMsMax: 0, + httpLatencyMsMax: 0, + stickyAssignments: 0, + stickyAssignmentRejections: 0, + placementAssignments: 0, + placementAssignmentRejections: 0, + stickyRejectionsByReason: {}, + placementRejectionsByReason: {}, + requestedRegions: {}, + selectedRegions: {}, + regionFallbacks: {}, + unavailableRegions: {}, + controlClosesByCode: {}, + spliceClosesByTrigger: {}, + controlRenewalLatenciesMs: [], + controlRenewalsByOutcome: {}, + controlActivityRecoveries: 0, + controlActivityRecoveryFailures: 0 +}) + +function percentile(values: number[], percentileRank: number): number { + if (values.length === 0) return 0 + const sorted = [...values].sort((left, right) => left - right) + return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0 +} + +export class RelayObservability implements RelayRuntimeObserver { + private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 }) + private deltas = emptyDeltas() + private timer: ReturnType | null = null + private lastFlushAt = 0 + private lastFlushedSafety = { + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0 + } + + constructor( + private readonly identity: { role: string; cellId: string; region: RelayRegion }, + private readonly write: MetricWriter = (entry) => console.log(JSON.stringify(entry)) + ) {} + + recordAuth(success: boolean): void { + if (success) this.deltas.authSuccesses++ + else this.deltas.authFailures++ + } + + recordForwardedBytes(bytes: number): void { + this.deltas.forwardedBytes += bytes + } + + recordHttp(durationMs: number): void { + this.deltas.httpLatencyMsMax = Math.max(this.deltas.httpLatencyMsMax, durationMs) + } + + recordReconnect(): void { + this.deltas.reconnects++ + } + + recordAssignmentAdmission(outcome: AssignmentAdmissionOutcome): void { + if (outcome === 'sticky') this.deltas.stickyAssignments++ + else if (outcome === 'sticky-rejected') this.deltas.stickyAssignmentRejections++ + else if (outcome === 'placement') this.deltas.placementAssignments++ + else this.deltas.placementAssignmentRejections++ + } + + recordAssignmentRejectionReason(lane: AssignmentAdmissionLane, reason: string): void { + const counts = + lane === 'sticky' + ? this.deltas.stickyRejectionsByReason + : this.deltas.placementRejectionsByReason + counts[reason] = (counts[reason] ?? 0) + 1 + } + + recordRegionRequest(region: RelayRegion | undefined): void { + increment(this.deltas.requestedRegions, region ?? 'unhinted') + } + + recordRegionSelection(input: { + targetRegion: RelayRegion + selectedRegion?: RelayRegion + fallback: boolean + }): void { + if (input.selectedRegion) increment(this.deltas.selectedRegions, input.selectedRegion) + else increment(this.deltas.unavailableRegions, input.targetRegion) + if (input.fallback) increment(this.deltas.regionFallbacks, input.targetRegion) + } + + recordSql(durationMs: number, success: boolean): void { + this.deltas.sqlQueries++ + if (!success) this.deltas.sqlFailures++ + this.deltas.sqlLatencyMsMax = Math.max(this.deltas.sqlLatencyMsMax, durationMs) + } + + recordControlRenewal(durationMs: number, outcome: ControlRenewalOutcome): void { + this.deltas.controlRenewalLatenciesMs.push(durationMs) + this.deltas.controlRenewalsByOutcome[outcome] = + (this.deltas.controlRenewalsByOutcome[outcome] ?? 0) + 1 + } + + recordControlActivityRecovery(success: boolean): void { + if (success) this.deltas.controlActivityRecoveries++ + else this.deltas.controlActivityRecoveryFailures++ + } + + recordReadiness(observation: RelayReadinessObservation): void { + this.write({ + severity: observation.ready ? 'INFO' : 'WARNING', + message: 'Orca Relay readiness check', + event: 'orca_relay_readiness_check', + metricVersion: 1, + ...this.identity, + ...observation + }) + } + + recordControlClose(code: number): void { + const key = String(code) + this.deltas.controlClosesByCode[key] = (this.deltas.controlClosesByCode[key] ?? 0) + 1 + } + + recordSpliceClose(trigger: string): void { + this.deltas.spliceClosesByTrigger[trigger] = + (this.deltas.spliceClosesByTrigger[trigger] ?? 0) + 1 + } + + start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void { + if (this.timer) return + this.eventLoop.enable() + this.timer = setInterval(() => this.flush(readCounts()), intervalMs) + this.timer.unref() + } + + stop(): void { + if (this.timer) clearInterval(this.timer) + this.timer = null + this.eventLoop.disable() + } + + regionalRehomeRuntimeSafety(): RegionalRehomeRuntimeSafety { + return { + observedAt: this.lastFlushAt, + sqlFailures: this.lastFlushedSafety.sqlFailures + this.deltas.sqlFailures, + reconnects: this.lastFlushedSafety.reconnects + this.deltas.reconnects, + controlActivityRecoveryFailures: + this.lastFlushedSafety.controlActivityRecoveryFailures + + this.deltas.controlActivityRecoveryFailures + } + } + + flush(counts: RelayProcessCounts): void { + this.lastFlushAt = Date.now() + const deltas = this.deltas + this.lastFlushedSafety = { + sqlFailures: deltas.sqlFailures, + reconnects: deltas.reconnects, + controlActivityRecoveryFailures: deltas.controlActivityRecoveryFailures + } + this.deltas = emptyDeltas() + const memory = process.memoryUsage() + const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000 + this.eventLoop.reset() + this.write({ + severity: 'INFO', + message: 'Orca Relay runtime metrics', + event: 'orca_relay_runtime_metrics', + metricVersion: 2, + role: this.identity.role, + cellId: this.identity.cellId, + region: this.identity.region, + ...counts, + forwardedBytesDelta: deltas.forwardedBytes, + authSuccessesDelta: deltas.authSuccesses, + authFailuresDelta: deltas.authFailures, + reconnectsDelta: deltas.reconnects, + stickyAssignmentsDelta: deltas.stickyAssignments, + stickyAssignmentRejectionsDelta: deltas.stickyAssignmentRejections, + placementAssignmentsDelta: deltas.placementAssignments, + placementAssignmentRejectionsDelta: deltas.placementAssignmentRejections, + stickyRejectionsByReasonDelta: deltas.stickyRejectionsByReason, + placementRejectionsByReasonDelta: deltas.placementRejectionsByReason, + requestedRegionsDelta: deltas.requestedRegions, + selectedRegionsDelta: deltas.selectedRegions, + regionFallbacksDelta: deltas.regionFallbacks, + unavailableRegionsDelta: deltas.unavailableRegions, + controlClosesByCodeDelta: deltas.controlClosesByCode, + spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger, + sqlQueriesDelta: deltas.sqlQueries, + sqlFailuresDelta: deltas.sqlFailures, + sqlLatencyMsMax: Number(deltas.sqlLatencyMsMax.toFixed(3)), + controlRenewalsByOutcomeDelta: deltas.controlRenewalsByOutcome, + controlRenewalsDelta: deltas.controlRenewalLatenciesMs.length, + controlRenewalSuccessesDelta: deltas.controlRenewalsByOutcome.renewed ?? 0, + controlRenewalLeaseMissesDelta: + deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0, + controlActivityRecoveriesDelta: deltas.controlActivityRecoveries, + controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures, + controlRenewalLatencyMsP50: Number( + percentile(deltas.controlRenewalLatenciesMs, 0.5).toFixed(3) + ), + controlRenewalLatencyMsP95: Number( + percentile(deltas.controlRenewalLatenciesMs, 0.95).toFixed(3) + ), + controlRenewalLatencyMsMax: Number( + Math.max(0, ...deltas.controlRenewalLatenciesMs).toFixed(3) + ), + httpLatencyMsMax: Number(deltas.httpLatencyMsMax.toFixed(3)), + heapUsedBytes: memory.heapUsed, + heapTotalBytes: memory.heapTotal, + eventLoopDelayMsP99: Number(p99.toFixed(3)) + }) + } +} + +function increment(counts: Record, key: string): void { + counts[key] = (counts[key] ?? 0) + 1 +} + +export function timedRelayOperation( + operation: () => Promise, + observe: (durationMs: number, success: boolean) => void, + isExpectedError: (error: unknown) => boolean = () => false +): Promise { + const startedAt = performance.now() + return operation().then( + (result) => { + observe(performance.now() - startedAt, true) + return result + }, + (error: unknown) => { + observe(performance.now() - startedAt, isExpectedError(error)) + throw error + } + ) +} diff --git a/cloud/apps/relay/src/relay-readiness.test.ts b/cloud/apps/relay/src/relay-readiness.test.ts new file mode 100644 index 00000000000..a85d9a46aa7 --- /dev/null +++ b/cloud/apps/relay/src/relay-readiness.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RelayDatabase } from './database.js' +import { + createRelayReadiness, + type RelayReadinessObservation +} from './relay-readiness.js' + +function database(query: () => Promise[]>): RelayDatabase { + return { + query, + queryLocked: query, + transaction: async (operation) => await operation(database(query)), + close: async () => {} + } +} + +describe('relay readiness', () => { + it('fails readiness while liveness remains independent of SQL and JWKS', async () => { + const jwksFailure = createRelayReadiness(database(async () => [{ ready: 1 }]), 'https://jwks', { + fetch: vi.fn(async () => new Response('', { status: 503 })) as typeof fetch, + cacheMs: 0 + }) + const sqlFailure = createRelayReadiness( + database(async () => { + throw new Error('sql down') + }), + 'https://jwks', + { + fetch: vi.fn(async () => new Response('{}', { status: 200 })) as typeof fetch, + cacheMs: 0 + } + ) + + expect(await jwksFailure()).toBe(false) + expect(await sqlFailure()).toBe(false) + }) + + it.each([ + { + name: 'JWKS HTTP failure', + fetch: vi.fn(async () => new Response('', { status: 503 })) as typeof fetch, + query: vi.fn(async () => [{ ready: 1 }]), + failure: 'jwks_http_failed' + }, + { + name: 'JWKS timeout', + fetch: vi.fn(async () => { + throw new DOMException('redacted', 'TimeoutError') + }) as typeof fetch, + query: vi.fn(async () => [{ ready: 1 }]), + failure: 'jwks_timed_out' + }, + { + name: 'JWKS fetch failure', + fetch: vi.fn(async () => { + throw new Error('redacted') + }) as typeof fetch, + query: vi.fn(async () => [{ ready: 1 }]), + failure: 'jwks_fetch_failed' + }, + { + name: 'SQL failure', + fetch: vi.fn(async () => new Response('{}', { status: 200 })) as typeof fetch, + query: vi.fn(async () => { + throw new Error('redacted') + }), + failure: 'sql_failed' + } + ])('reports a safe reason for $name', async ({ fetch, query, failure }) => { + const observations: RelayReadinessObservation[] = [] + const ready = createRelayReadiness(database(query), 'https://jwks', { + fetch, + cacheMs: 0, + observe: (observation) => observations.push(observation) + }) + + expect(await ready()).toBe(false) + expect(observations).toEqual([ + expect.objectContaining({ ready: false, failure }) + ]) + expect(JSON.stringify(observations)).not.toContain('redacted') + if (failure.startsWith('jwks_')) expect(query).not.toHaveBeenCalled() + }) + + it('reports the initial success but not healthy repeats or cached reads', async () => { + const observations: RelayReadinessObservation[] = [] + let now = 100 + const ready = createRelayReadiness(database(async () => [{ ready: 1 }]), 'https://jwks', { + fetch: vi.fn(async () => new Response('{}', { status: 200 })) as typeof fetch, + cacheMs: 10_000, + now: () => now, + observe: (observation) => observations.push(observation) + }) + + expect(await ready()).toBe(true) + now += 1_000 + expect(await ready()).toBe(true) + now += 10_000 + expect(await ready()).toBe(true) + expect(observations).toEqual([ + { + ready: true, + jwksLatencyMs: 0, + sqlLatencyMs: 0, + totalLatencyMs: 0 + } + ]) + }) +}) diff --git a/cloud/apps/relay/src/relay-readiness.ts b/cloud/apps/relay/src/relay-readiness.ts new file mode 100644 index 00000000000..0b7303367f1 --- /dev/null +++ b/cloud/apps/relay/src/relay-readiness.ts @@ -0,0 +1,89 @@ +import type { RelayDatabase } from './database.js' + +export type RelayReadinessFailure = + | 'jwks_fetch_failed' + | 'jwks_http_failed' + | 'jwks_timed_out' + | 'sql_failed' + +export type RelayReadinessObservation = { + ready: boolean + failure?: RelayReadinessFailure + jwksLatencyMs: number + sqlLatencyMs: number + totalLatencyMs: number +} + +type RelayReadinessOptions = { + fetch?: typeof fetch + timeoutMs?: number + cacheMs?: number + now?: () => number + observe?: (observation: RelayReadinessObservation) => void +} + +function fetchFailure(error: unknown): RelayReadinessFailure { + return error instanceof Error && error.name === 'TimeoutError' + ? 'jwks_timed_out' + : 'jwks_fetch_failed' +} + +export function createRelayReadiness( + database: RelayDatabase, + jwksUrl: string, + options: RelayReadinessOptions = {} +): () => Promise { + const fetchImpl = options.fetch ?? fetch + const timeoutMs = options.timeoutMs ?? 2_000 + const cacheMs = options.cacheMs ?? 10_000 + const now = options.now ?? Date.now + let cachedAt = Number.NEGATIVE_INFINITY + let cached = false + let lastObservedReady: boolean | undefined + + return async () => { + if (now() - cachedAt < cacheMs) return cached + const startedAt = now() + let jwksCompletedAt = startedAt + let sqlStartedAt = startedAt + let failure: RelayReadinessFailure | undefined + try { + let response: Response + try { + response = await fetchImpl(jwksUrl, { signal: AbortSignal.timeout(timeoutMs) }) + } catch (error) { + failure = fetchFailure(error) + throw error + } finally { + jwksCompletedAt = now() + } + if (!response.ok) { + failure = 'jwks_http_failed' + throw new Error(failure) + } + sqlStartedAt = now() + try { + await database.query('SELECT 1 AS ready') + } catch (error) { + failure = 'sql_failed' + throw error + } + } catch { + // The load balancer only needs the boolean; the safe reason is emitted below. + } + const completedAt = now() + cached = failure === undefined + cachedAt = completedAt + if (!cached || cached !== lastObservedReady) { + options.observe?.({ + ready: cached, + ...(failure ? { failure } : {}), + jwksLatencyMs: Math.max(0, jwksCompletedAt - startedAt), + sqlLatencyMs: failure?.startsWith('jwks_') ? 0 : Math.max(0, completedAt - sqlStartedAt), + totalLatencyMs: Math.max(0, completedAt - startedAt) + }) + } + lastObservedReady = cached + return cached + } +} diff --git a/cloud/apps/relay/src/relay-region-app.test.ts b/cloud/apps/relay/src/relay-region-app.test.ts new file mode 100644 index 00000000000..30cf3bf3e26 --- /dev/null +++ b/cloud/apps/relay/src/relay-region-app.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RelayConfig } from './config.js' + +const fakes = vi.hoisted(() => ({ + verifyRelayToken: vi.fn(async (token: string) => ({ sub: 'user-1', relayHostId: token })) +})) + +vi.mock('./relay-token-verifier.js', () => ({ + createRelayTokenVerifier: () => fakes.verifyRelayToken, + readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null +})) + +import { createRelayApp } from './app.js' + +describe('Relay region API', () => { + it('passes a valid preference to placement and records coarse outcomes', async () => { + const assign = vi.fn(async () => ({ + userId: 'user-1', + relayHostId: 'asiahost00000001', + cellId: 'asia-c1', + cellUrl: 'https://asia-c1.relay.example.test', + region: 'asia-east2' as const, + assignmentEpoch: 1, + leaseExpiresAt: Date.now() + 300_000 + })) + const recordRegionRequest = vi.fn() + const recordRegionSelection = vi.fn() + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, resolve: vi.fn(async () => null) } as never, + drain: vi.fn(), + ready: vi.fn(async () => true), + recordRegionRequest, + recordRegionSelection + }) + + const response = await app.request( + '/v1/assign', + assignmentRequest('asiahost00000001', { preferredRegion: 'asia-east2' }) + ) + + expect(response.status).toBe(200) + expect(assign).toHaveBeenCalledWith( + { userId: 'user-1', relayHostId: 'asiahost00000001' }, + 'asia-east2', + 'asia-east2' + ) + expect(recordRegionRequest).toHaveBeenCalledWith('asia-east2') + expect(recordRegionSelection).toHaveBeenCalledWith({ + targetRegion: 'asia-east2', + selectedRegion: 'asia-east2', + fallback: false + }) + }) + + it('keeps the preference for observation while the kill switch places US-first', async () => { + const assign = vi.fn(async () => ({ + userId: 'user-1', + relayHostId: 'killhost00000001', + cellId: 'us-c1', + cellUrl: 'https://us-c1.relay.example.test', + region: 'us-central1' as const, + assignmentEpoch: 1, + leaseExpiresAt: Date.now() + 300_000 + })) + const app = createRelayApp(config({ regionalPlacementEnabled: false }), { + store: {} as never, + assignments: { assign, resolve: vi.fn(async () => null) } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const response = await app.request( + '/v1/assign', + assignmentRequest('killhost00000001', { preferredRegion: 'asia-east2' }) + ) + + expect(response.status).toBe(200) + expect(assign).toHaveBeenCalledWith( + { userId: 'user-1', relayHostId: 'killhost00000001' }, + 'asia-east2', + 'us-central1' + ) + }) + + it('exposes only the store-provided healthy catalog from directors', async () => { + const regionCatalog = vi.fn(async () => [ + { region: 'us-central1' as const, probeOrigins: ['https://us.relay.example.test'] } + ]) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { regionCatalog } as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + + const response = await app.request('/v1/regions') + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + v: 1, + regions: [ + { region: 'us-central1', probeOrigins: ['https://us.relay.example.test'] } + ] + }) + + const burst = await Promise.all( + Array.from({ length: 50 }, () => app.request('/v1/regions')) + ) + expect(burst.every(({ status }) => status === 200)).toBe(true) + expect(regionCatalog).toHaveBeenCalledOnce() + }) +}) + +function assignmentRequest(relayHostId: string, extra: Record): RequestInit { + return { + method: 'POST', + headers: { + authorization: `Bearer ${relayHostId}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, relayHostId, ...extra }) + } +} + +function config(overrides: Partial = {}): RelayConfig { + return { + port: 8080, + publicUrl: 'https://relay.example.test', + cellUrl: 'https://relay.example.test', + region: 'us-central1', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'), + role: 'director', + cellId: 'director', + cells: [], + adminAudience: 'https://relay.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + runtimeServiceAccount: 'runtime@example.test', + adminJwksUrl: 'https://auth.example.test/jwks', + databasePoolMax: 3, + publicAssignmentsEnabled: true, + regionalPlacementEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './data', + ...overrides + } +} diff --git a/cloud/apps/relay/src/relay-region-placement.test.ts b/cloud/apps/relay/src/relay-region-placement.test.ts new file mode 100644 index 00000000000..76cf5479a41 --- /dev/null +++ b/cloud/apps/relay/src/relay-region-placement.test.ts @@ -0,0 +1,196 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayCellConfig } from './config.js' +import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js' + +const CELLS: RelayCellConfig[] = [ + { + id: 'us-c1', + url: 'https://us-c1.relay.example.com', + region: 'us-central1', + capacityRequests: 100 + }, + { + id: 'asia-c1', + url: 'https://asia-c1.relay.example.com', + region: 'asia-east2', + capacityRequests: 100 + }, + { + id: 'asia-c2', + url: 'https://asia-c2.relay.example.com', + region: 'asia-east2', + capacityRequests: 100 + }, + { + id: 'asia-c3', + url: 'https://asia-c3.relay.example.com', + region: 'asia-east2', + capacityRequests: 100 + } +] + +describe('Relay regional placement', () => { + let database: RelayDatabase + let now: number + let store: RelayAssignmentStore + + beforeEach(async () => { + database = await openInMemoryRelayDatabase() + now = 1_000 + store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells(CELLS) + }) + + afterEach(async () => await database.close()) + + it('prefers the requested region and keeps unhinted placement US-first', async () => { + await expect( + store.assign({ userId: 'asia-user', relayHostId: 'asiahost00000001' }, 'asia-east2') + ).resolves.toMatchObject({ cellId: 'asia-c1', region: 'asia-east2' }) + await expect( + store.assign({ userId: 'us-user', relayHostId: 'ushost0000000001' }) + ).resolves.toMatchObject({ cellId: 'us-c1', region: 'us-central1' }) + }) + + it('falls back globally only when the target region has no safe general cell', async () => { + await store.configureCell(CELLS[1]!, 'migration-only') + await store.configureCell(CELLS[2]!, 'migration-only') + await store.configureCell(CELLS[3]!, 'migration-only') + + await expect( + store.assign({ userId: 'fallback-user', relayHostId: 'fallbackhost0001' }, 'asia-east2') + ).resolves.toMatchObject({ cellId: 'us-c1', region: 'us-central1' }) + }) + + it('preserves sticky assignments while updating only explicit preferences', async () => { + const identity = { userId: 'sticky-user', relayHostId: 'stickyhost000001' } + await expect(store.assign(identity)).resolves.toMatchObject({ cellId: 'us-c1' }) + now = 2_000 + await expect(store.assign(identity, 'asia-east2')).resolves.toMatchObject({ cellId: 'us-c1' }) + now = 3_000 + await store.assign(identity) + + expect( + await database.query( + `SELECT preferred_region, observed_at FROM relay_assignment_region_preferences + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ preferred_region: 'asia-east2', observed_at: 2_000 }]) + }) + + it('uses the explicit placement region as the server-side kill switch', async () => { + await expect( + store.assign( + { userId: 'kill-user', relayHostId: 'killhost00000001' }, + 'asia-east2', + 'us-central1' + ) + ).resolves.toMatchObject({ cellId: 'us-c1', region: 'us-central1' }) + expect(await database.query(`SELECT preferred_region FROM relay_assignment_region_preferences`)) + .toEqual([{ preferred_region: 'asia-east2' }]) + }) + + it('returns at most two deterministic healthy general probe origins per region', async () => { + for (const [index, cell] of CELLS.entries()) { + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + region: cell.region, + cellIncarnation: `00000000-0000-4000-8000-${String(index + 1).padStart(12, '0')}`, + startedAt: 1, + ready: true, + observedRequests: 0 + }) + } + await store.configureCell(CELLS[2]!, 'migration-only') + + await expect(store.regionCatalog()).resolves.toEqual([ + { region: 'us-central1', probeOrigins: ['https://us-c1.relay.example.com'] }, + { + region: 'asia-east2', + probeOrigins: [ + 'https://asia-c1.relay.example.com', + 'https://asia-c3.relay.example.com' + ] + } + ]) + + await database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, ['asia-c3']) + await expect(store.regionCatalog()).resolves.toEqual([ + { region: 'us-central1', probeOrigins: ['https://us-c1.relay.example.com'] }, + { region: 'asia-east2', probeOrigins: ['https://asia-c1.relay.example.com'] } + ]) + }) + + it('rejects a heartbeat whose explicit region conflicts with registration', async () => { + await expect( + store.recordCellHeartbeat({ + cellId: 'asia-c1', + cellUrl: 'https://asia-c1.relay.example.com', + region: 'us-central1', + cellIncarnation: '00000000-0000-4000-8000-000000000001', + startedAt: 1, + ready: true, + observedRequests: 0 + }) + ).rejects.toThrow('cell_region_mismatch') + }) + + it('defaults cells inserted by an old process after startup to US', async () => { + const oldCell = { + id: 'old-us-cell', + url: 'https://old-us-cell.relay.example.com', + capacityRequests: 100 + } + await database.query( + `INSERT INTO relay_cells + (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) + VALUES (?, ?, 1, 100, 0, 0, ?, ?)`, + [oldCell.id, oldCell.url, now, now] + ) + await database.query( + `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at) + VALUES (?, 'general', ?)`, + [oldCell.id, now] + ) + await Promise.all(CELLS.map((cell) => store.configureCell(cell, 'migration-only'))) + + const identity = { userId: 'old-user', relayHostId: 'oldhost000000001' } + await expect(store.assign(identity)).resolves.toMatchObject({ + cellId: oldCell.id, + region: 'us-central1' + }) + await expect(store.resolve(identity)).resolves.toMatchObject({ + cellId: oldCell.id, + region: 'us-central1' + }) + await expect( + store.recordCellHeartbeat({ + cellId: oldCell.id, + cellUrl: oldCell.url, + cellIncarnation: '00000000-0000-4000-8000-000000000099', + startedAt: 1, + ready: true, + observedRequests: 0 + }) + ).resolves.toBeUndefined() + }) + + it('expires identity-linked region preferences after 30 days', async () => { + const identity = { userId: 'expiry-user', relayHostId: 'expiryhost000001' } + await store.assign(identity, 'asia-east2') + now += 30 * 24 * 60 * 60_000 + 1 + + await expect(store.releaseExpiredRegionPreferences()).resolves.toBe(1) + await expect( + database.query( + `SELECT preferred_region FROM relay_assignment_region_preferences + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + ).resolves.toEqual([]) + }) +}) diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts new file mode 100644 index 00000000000..a14240cfa6a --- /dev/null +++ b/cloud/apps/relay/src/relay-server.ts @@ -0,0 +1,533 @@ +import { createAdaptorServer } from '@hono/node-server' +import { + hasAdmissionCapacity, + HostDataAuthSchema, + RELAY_ADMISSION_BUDGETS, + RELAY_CLOSE_CODE, + RELAY_DEFAULT_REGION, + RELAY_PROTOCOL_LIMITS, + RelayAuthSchema +} from '@orca-cloud/relay-contract' +import type { IncomingMessage } from 'node:http' +import { randomUUID } from 'node:crypto' +import { performance } from 'node:perf_hooks' +import { WebSocketServer } from 'ws' +import type WebSocket from 'ws' +import type { RawData } from 'ws' +import { createRelayApp } from './app.js' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import { RelayCredentialStore } from './credential-store.js' +import type { RelayDatabase } from './database.js' +import { HostSessionRegistry } from './host-session-registry.js' +import { observeRelayDatabase } from './observed-relay-database.js' +import { RelayObservability } from './relay-observability.js' +import { + RelayConnectionLedger, + type RelayConnectionUpgrade +} from './relay-connection-ledger.js' +import { createRelayReadiness } from './relay-readiness.js' +import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js' +import { closeRelayWebSocket } from './relay-websocket-close.js' +import { ProcessQueuedByteBudget } from './splice-forwarder.js' + +function rejectUpgrade(socket: NodeJS.WritableStream, status: number, message: string): void { + socket.write(`HTTP/1.1 ${status} ${message}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`) + if ('destroy' in socket && typeof socket.destroy === 'function') socket.destroy() +} + +function noDelay(socket: WebSocket): void { + const transport = (socket as WebSocket & { _socket?: { setNoDelay: (enabled: boolean) => void } }) + ._socket + transport?.setNoDelay(true) +} + +// Why: a ws receiver error (oversize or malformed frame) with no 'error' +// listener throws process-wide; ws itself already closes the socket after +// emitting it, so logging is all that is left to do. +function guardSocketErrors(socket: WebSocket, kind: string): void { + socket.on('error', (error) => { + console.warn(`[orca-relay] ${kind} socket error: ${error.message}`) + }) +} + +function admissionSource(request: IncomingMessage): string { + const forwarded = request.headers['x-forwarded-for'] + const chain = (Array.isArray(forwarded) ? forwarded.join(',') : forwarded ?? '') + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) + // Google Front End appends client and load-balancer addresses after any + // caller-supplied values, so only the penultimate hop is trustworthy. + return chain.length >= 2 ? chain.at(-2)! : (request.socket.remoteAddress ?? 'unknown') +} + +function firstPayload(raw: RawData, expectedType: string): unknown { + try { + const parsed = JSON.parse(raw.toString()) as Record + if (parsed.type !== expectedType) return null + const { type: _type, ...rest } = parsed + return rest + } catch { + return null + } +} + +export function createRelayServer( + config: RelayConfig, + database: RelayDatabase, + options: { + now?: () => number + connectionLedgerLimits?: { hardCap: number; controlReserve: number } + cellIncarnation?: string + } = {} +) { + const cellIncarnation = options.cellIncarnation ?? randomUUID() + const observability = new RelayObservability({ + role: config.role, + cellId: config.cellId, + region: config.region ?? RELAY_DEFAULT_REGION + }) + const observedDatabase = observeRelayDatabase(database, observability) + const controls = new WebSocketServer({ + noServer: true, + clientTracking: false, + perMessageDeflate: false, + maxPayload: 1024 * 1024 + }) + const verifyRelayToken = createRelayTokenVerifier(config) + const store = new RelayCredentialStore(observedDatabase, options.now) + const assignments = new RelayAssignmentStore(observedDatabase, options.now, { + requireLiveCells: config.role === 'director', + recordControlRenewal: (durationMs, outcome) => + observability.recordControlRenewal?.(durationMs, outcome) + }) + const ready = createRelayReadiness(observedDatabase, config.jwksUrl, { + observe: (observation) => observability.recordReadiness(observation) + }) + const queuedBytes = new ProcessQueuedByteBudget() + const sessions = new HostSessionRegistry( + config, + verifyRelayToken, + store, + assignments, + queuedBytes, + observability, + options.now + ) + const app = createRelayApp(config, { + store, + assignments, + drain: (graceMs) => sessions.drain(graceMs), + drainHost: (input) => sessions.drainHost(input), + regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null, + cellIncarnation, + isDraining: () => sessions.isDraining(), + runtimeCounts: () => runtimeCounts(), + ready, + recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome), + recordAssignmentRejectionReason: (lane, reason) => + observability.recordAssignmentRejectionReason?.(lane, reason), + recordRegionRequest: (region) => observability.recordRegionRequest?.(region), + recordRegionSelection: (input) => observability.recordRegionSelection?.(input) + }) + const observedFetch: typeof app.fetch = async (...args) => { + const startedAt = performance.now() + try { + return await app.fetch(...args) + } finally { + observability.recordHttp(performance.now() - startedAt) + } + } + const server = createAdaptorServer({ fetch: observedFetch }) + const clients = new WebSocketServer({ + noServer: true, + clientTracking: false, + perMessageDeflate: false, + maxPayload: RELAY_PROTOCOL_LIMITS.maxFrameBytes + }) + const dataSockets = new WebSocketServer({ + noServer: true, + clientTracking: false, + perMessageDeflate: false, + maxPayload: RELAY_PROTOCOL_LIMITS.maxFrameBytes + }) + let preAuthConnections = 0 + let totalConnections = 0 + const configuredConnectionLimits = + config.connectionHardCap === undefined + ? null + : (options.connectionLedgerLimits ?? { + hardCap: config.connectionHardCap, + controlReserve: RELAY_ADMISSION_BUDGETS.reservedHostControls + }) + const connectionLedger = + configuredConnectionLimits === null + ? null + : new RelayConnectionLedger( + configuredConnectionLimits.hardCap, + configuredConnectionLimits.controlReserve + ) + const preAuthBySource = new Map() + const preAuthAttemptsBySource = new Map() + + const admit = (source: string): boolean => { + const now = Date.now() + const currentWindow = preAuthAttemptsBySource.get(source) + const attempts = + !currentWindow || now - currentWindow.windowStartedAt >= 60_000 + ? { windowStartedAt: now, count: 0 } + : currentWindow + const sourceCount = preAuthBySource.get(source) ?? 0 + if ( + !hasAdmissionCapacity({ + totalRequests: + connectionLedger?.counts().physicalConnections ?? totalConnections, + preAuthConnections, + sourcePreAuthConnections: sourceCount, + totalRequestCeiling: + configuredConnectionLimits === null + ? undefined + : configuredConnectionLimits.hardCap - configuredConnectionLimits.controlReserve + }) || + attempts.count >= RELAY_ADMISSION_BUDGETS.maxPreAuthAttemptsPerSourcePerMinute + ) { + return false + } + attempts.count++ + preAuthAttemptsBySource.delete(source) + preAuthAttemptsBySource.set(source, attempts) + // A bounded LRU keeps source churn from becoming its own memory attack. + if (preAuthAttemptsBySource.size > 4_096) { + preAuthAttemptsBySource.delete(preAuthAttemptsBySource.keys().next().value!) + } + preAuthConnections++ + preAuthBySource.set(source, sourceCount + 1) + return true + } + const authenticated = (source: string): void => { + preAuthConnections = Math.max(0, preAuthConnections - 1) + const next = (preAuthBySource.get(source) ?? 1) - 1 + if (next <= 0) preAuthBySource.delete(source) + else preAuthBySource.set(source, next) + } + + const trackConnection = ( + socket: WebSocket, + upgrade: RelayConnectionUpgrade | null + ): void => { + if (upgrade) { + upgrade.promote(socket) + return + } + totalConnections++ + socket.once('close', () => { + totalConnections = Math.max(0, totalConnections - 1) + }) + } + + const awaitFirstFrame = ( + socket: WebSocket, + source: string, + callback: (raw: RawData) => Promise + ): void => { + let finished = false + const timer = setTimeout(() => { + if (finished) return + finished = true + authenticated(source) + observability.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'first frame timeout') + }, RELAY_PROTOCOL_LIMITS.firstFrameDeadlineMs) + socket.once('message', (raw, binary) => { + if (finished) return + finished = true + clearTimeout(timer) + authenticated(source) + if (binary) { + observability.recordAuth(false) + socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'first frame must be text') + return + } + void callback(raw).catch((error: unknown) => { + console.warn( + '[orca-relay] first frame handler failed', + error instanceof Error ? error.message : '' + ) + closeRelayWebSocket( + socket, + RELAY_CLOSE_CODE.LIMIT_EXCEEDED, + 'relay temporarily unavailable' + ) + }) + }) + socket.once('close', () => { + if (!finished) { + finished = true + clearTimeout(timer) + authenticated(source) + } + }) + } + + server.on('upgrade', (request, socket, head) => { + const url = new URL(request.url ?? '/', config.publicUrl) + const source = admissionSource(request) + if (url.search) { + rejectUpgrade(socket, 400, 'Bad Request') + return + } + if (url.pathname.startsWith('/v1/connect/')) { + const hostId = decodeURIComponent(url.pathname.slice('/v1/connect/'.length)) + if (!/^[A-Za-z0-9_-]{16}$/.test(hostId)) { + rejectUpgrade(socket, 429, 'Too Many Requests') + return + } + const phoneAdmission = connectionLedger?.tryReservePhone() ?? null + if (connectionLedger && !phoneAdmission) { + rejectUpgrade(socket, 503, 'Service Unavailable') + return + } + if (!admit(source)) { + phoneAdmission?.upgrade.release() + phoneAdmission?.hostData.release() + rejectUpgrade(socket, 429, 'Too Many Requests') + return + } + const releasePhoneUpgrade = (): void => { + authenticated(source) + phoneAdmission?.upgrade.release() + phoneAdmission?.hostData.release() + } + socket.once('close', releasePhoneUpgrade) + try { + clients.handleUpgrade(request, socket, head, (webSocket) => { + socket.off('close', releasePhoneUpgrade) + trackConnection(webSocket, phoneAdmission?.upgrade ?? null) + guardSocketErrors(webSocket, 'client') + if (phoneAdmission) { + webSocket.once('close', () => phoneAdmission.hostData.release()) + } + noDelay(webSocket) + awaitFirstFrame(webSocket, source, async (raw) => { + const auth = RelayAuthSchema.safeParse(firstPayload(raw, 'relay-auth')) + if (!auth.success) { + phoneAdmission?.hostData.release() + observability.recordAuth(false) + webSocket.send( + JSON.stringify({ type: 'relay-hello', ok: false, code: RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL }) + ) + webSocket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid relay auth') + return + } + if (config.role === 'director') { + const invite = await store.resolveInviteForMove(hostId, auth.data.credential) + const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null + // Released combined-service invites gain their first durable cell assignment here. + const assignment = identity + ? (await assignments.resolve(identity)) ?? (await assignments.assign(identity)) + : null + if (!invite || !assignment) { + phoneAdmission?.hostData.release() + observability.recordAuth(false) + webSocket.send( + JSON.stringify({ + type: 'relay-hello', + ok: false, + code: RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL + }) + ) + webSocket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid invite') + return + } + phoneAdmission?.hostData.release() + observability.recordAuth(true) + webSocket.send( + JSON.stringify({ + type: 'relay-moved', + v: 1, + cellUrl: assignment.cellUrl, + assignmentEpoch: assignment.assignmentEpoch + }) + ) + webSocket.close(RELAY_CLOSE_CODE.DRAINING, 'connect to assigned cell') + return + } + await sessions.acceptClient( + webSocket, + hostId, + auth.data.credential, + phoneAdmission?.hostData + ) + }) + }) + } catch { + socket.off('close', releasePhoneUpgrade) + releasePhoneUpgrade() + socket.destroy() + } + return + } + if (url.pathname.startsWith('/v1/host/data/')) { + if (config.role === 'director') { + rejectUpgrade(socket, 404, 'Not Found') + return + } + const connId = decodeURIComponent(url.pathname.slice('/v1/host/data/'.length)) + if (!connId || connId.length > 128) { + rejectUpgrade(socket, 429, 'Too Many Requests') + return + } + const dataUpgrade = connectionLedger?.tryReserveHostData(connId) ?? null + if (connectionLedger && !dataUpgrade) { + rejectUpgrade(socket, 503, 'Service Unavailable') + return + } + if (!admit(source)) { + dataUpgrade?.release() + rejectUpgrade(socket, 429, 'Too Many Requests') + return + } + const releaseDataUpgrade = (): void => { + authenticated(source) + dataUpgrade?.release() + } + socket.once('close', releaseDataUpgrade) + try { + dataSockets.handleUpgrade(request, socket, head, (webSocket) => { + socket.off('close', releaseDataUpgrade) + trackConnection(webSocket, dataUpgrade) + guardSocketErrors(webSocket, 'host-data') + noDelay(webSocket) + awaitFirstFrame(webSocket, source, async (raw) => { + const auth = HostDataAuthSchema.safeParse(firstPayload(raw, 'host-data-auth')) + if (!auth.success) { + observability.recordAuth(false) + webSocket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host data auth') + return + } + const accepted = await sessions.acceptHostData( + webSocket, + connId, + auth.data.connTicket, + auth.data.generation + ) + if (accepted) dataUpgrade?.commitHostData() + }) + }) + } catch { + socket.off('close', releaseDataUpgrade) + releaseDataUpgrade() + socket.destroy() + } + return + } + if (url.pathname !== '/v1/host/control') { + rejectUpgrade(socket, 404, 'Not Found') + return + } + if (config.role === 'director') { + rejectUpgrade(socket, 404, 'Not Found') + return + } + const bearer = readBearer(request.headers.authorization) + if (!bearer) { + observability.recordAuth(false) + rejectUpgrade(socket, 401, 'Unauthorized') + return + } + void verifyRelayToken(bearer).then((identity) => { + if (socket.destroyed) return + if (!identity) { + observability.recordAuth(false) + rejectUpgrade(socket, 401, 'Unauthorized') + return + } + const isRebind = sessions.hasActiveControl({ + userId: identity.sub, + relayHostId: identity.relayHostId + }) + const controlUpgrade = connectionLedger?.tryReserveControl(isRebind) ?? null + if ( + (connectionLedger && !controlUpgrade) || + (!connectionLedger && totalConnections >= RELAY_ADMISSION_BUDGETS.cloudRunConcurrency) + ) { + rejectUpgrade( + socket, + connectionLedger ? 503 : 429, + connectionLedger ? 'Service Unavailable' : 'Too Many Requests' + ) + return + } + // Register the release before anything else can throw: the outer catch + // destroys the socket, so a close-registered release cannot leak the + // reserved connection unit. + const releaseControlUpgrade = (): void => controlUpgrade?.release() + socket.once('close', releaseControlUpgrade) + observability.recordAuth(true) + try { + controls.handleUpgrade(request, socket, head, (webSocket) => { + socket.off('close', releaseControlUpgrade) + trackConnection(webSocket, controlUpgrade) + guardSocketErrors(webSocket, 'control') + noDelay(webSocket) + sessions.acceptControl( + webSocket, + identity, + controlUpgrade?.inclusionWatermark + ) + }) + } catch { + socket.off('close', releaseControlUpgrade) + releaseControlUpgrade() + socket.destroy() + } + }).catch((error: unknown) => { + // A throw in the upgrade handling above must cost this socket, not the process. + console.warn( + `[orca-relay] control upgrade failed: ${error instanceof Error ? error.message : 'unknown'}` + ) + socket.destroy() + }) + }) + + server.on('close', () => { + controls.close() + clients.close() + dataSockets.close() + }) + const runtimeCounts = () => { + const ledgerCounts = connectionLedger?.counts() + return { + totalConnections: ledgerCounts?.physicalConnections ?? totalConnections, + preAuthConnections, + ...sessions.runtimeCounts(), + queuedBytes: queuedBytes.current(), + ...(ledgerCounts + ? { + inFlightConnections: ledgerCounts.inFlightConnections, + reservedConnectionUnits: ledgerCounts.reservedConnectionUnits, + enforcedConnectionUnits: ledgerCounts.enforcedConnectionUnits + } + : {}) + } + } + const connectionSnapshot = () => connectionLedger?.snapshot() + return { + server, + sessions, + store, + assignments, + queuedBytes, + observability, + runtimeCounts, + connectionSnapshot, + ready, + cellIncarnation + } +} + +export function closeWithDrain(socket: WebSocket, graceMs: number): void { + socket.send(JSON.stringify({ type: 'drain', graceMs, recovery: 'resolve-director' })) + socket.close(RELAY_CLOSE_CODE.DRAINING, 'resolve configured director') +} diff --git a/cloud/apps/relay/src/relay-token-verifier.ts b/cloud/apps/relay/src/relay-token-verifier.ts new file mode 100644 index 00000000000..d3dc5300a44 --- /dev/null +++ b/cloud/apps/relay/src/relay-token-verifier.ts @@ -0,0 +1,35 @@ +import { createRemoteJWKSet, jwtVerify } from 'jose' +import { z } from 'zod' +import type { RelayConfig } from './config.js' + +const ClaimsSchema = z.object({ + sub: z.string().min(1), + prof: z.string().min(1), + org: z.string().min(1).optional(), + relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/), + purpose: z.literal('host-control'), + exp: z.number().int().positive() +}) + +export type RelayTokenClaims = z.infer + +export function createRelayTokenVerifier(config: RelayConfig): (token: string) => Promise { + const jwks = createRemoteJWKSet(new URL(config.jwksUrl)) + return async (token) => { + try { + const verified = await jwtVerify(token, jwks, { + issuer: config.authIssuer, + audience: config.authAudience, + algorithms: ['ES256'] + }) + return ClaimsSchema.parse(verified.payload) + } catch { + return null + } + } +} + +export function readBearer(value: string | undefined): string | null { + const match = /^Bearer ([^\s]+)$/.exec(value ?? '') + return match?.[1] ?? null +} diff --git a/cloud/apps/relay/src/relay-websocket-close.test.ts b/cloud/apps/relay/src/relay-websocket-close.test.ts new file mode 100644 index 00000000000..222da3f1a7e --- /dev/null +++ b/cloud/apps/relay/src/relay-websocket-close.test.ts @@ -0,0 +1,44 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type WebSocket from 'ws' +import { closeRelayWebSocket } from './relay-websocket-close.js' + +class FakeSocket extends EventEmitter { + readonly OPEN = 1 + readonly CLOSING = 2 + readonly CLOSED = 3 + readyState = this.OPEN + readonly close = vi.fn(() => { + this.readyState = this.CLOSING + }) + readonly terminate = vi.fn(() => { + this.readyState = this.CLOSED + this.emit('close') + }) +} + +describe('relay WebSocket close', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + it('terminates a peer that does not complete the close handshake', () => { + const socket = new FakeSocket() + closeRelayWebSocket(socket as unknown as WebSocket, 4408, 'relay draining') + + expect(socket.close).toHaveBeenCalledWith(4408, 'relay draining') + vi.advanceTimersByTime(999) + expect(socket.terminate).not.toHaveBeenCalled() + vi.advanceTimersByTime(1) + expect(socket.terminate).toHaveBeenCalledOnce() + }) + + it('cancels forced termination after the peer closes', () => { + const socket = new FakeSocket() + closeRelayWebSocket(socket as unknown as WebSocket, 4408, 'relay draining') + socket.readyState = socket.CLOSED + socket.emit('close') + vi.runAllTimers() + + expect(socket.terminate).not.toHaveBeenCalled() + }) +}) diff --git a/cloud/apps/relay/src/relay-websocket-close.ts b/cloud/apps/relay/src/relay-websocket-close.ts new file mode 100644 index 00000000000..d4a7f61a3ee --- /dev/null +++ b/cloud/apps/relay/src/relay-websocket-close.ts @@ -0,0 +1,22 @@ +import type WebSocket from 'ws' + +const RELAY_WEBSOCKET_FORCE_CLOSE_MS = 1_000 +const forceCloseTimers = new WeakMap>() + +export function closeRelayWebSocket(socket: WebSocket, code: number, reason: string): void { + if (socket.readyState === socket.CLOSED) return + if (!forceCloseTimers.has(socket)) { + const timer = setTimeout(() => { + forceCloseTimers.delete(socket) + if (socket.readyState !== socket.CLOSED) socket.terminate() + }, RELAY_WEBSOCKET_FORCE_CLOSE_MS) + timer.unref() + forceCloseTimers.set(socket, timer) + socket.once('close', () => { + const pending = forceCloseTimers.get(socket) + if (pending) clearTimeout(pending) + forceCloseTimers.delete(socket) + }) + } + if (socket.readyState === socket.OPEN) socket.close(code, reason) +} diff --git a/cloud/apps/relay/src/relay.blackbox.test.ts b/cloud/apps/relay/src/relay.blackbox.test.ts new file mode 100644 index 00000000000..38134213e76 --- /dev/null +++ b/cloud/apps/relay/src/relay.blackbox.test.ts @@ -0,0 +1,2622 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { createHash, createHmac } from 'node:crypto' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type Server } from 'node:http' +import { createServer as createNetServer } from 'node:net' +import { dirname, resolve } from 'node:path' +import { tmpdir } from 'node:os' +import { fileURLToPath } from 'node:url' +import { exportJWK, generateKeyPair, jwtVerify, SignJWT } from 'jose' +import { + buildHostProofMacInput, + HOST_CHALLENGE_PLAINTEXT_DOMAIN +} from '@orca-cloud/relay-contract' +import nacl from 'tweetnacl' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import WebSocket from 'ws' +import type { RawData } from 'ws' +import { RelayAssignmentStore } from './assignment-store.js' +import { + encodeMembership, + type CellAdmissionMembership +} from './cell-admission-selector.js' +import { openRelayDatabase } from './database.js' + +const appDirectory = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const assignmentKey = 'test-assignment-key-with-at-least-32-bytes' +let relayProcess: ChildProcess +let jwksServer: Server +let relayUrl: string +let issuer: string +let privateKey: Awaited>['privateKey'] +let adminPrivateKey: Awaited>['privateKey'] +let relayDataDirectory: string +let adminAudience: string +let forwardedSourceSequence = 0 + +function forwardedHeaders(): Record { + forwardedSourceSequence++ + return { + 'x-forwarded-for': `spoofed, 192.0.2.${forwardedSourceSequence}, 35.191.0.1` + } +} + +async function unusedPort(): Promise { + const server = createNetServer() + await new Promise((resolveListen) => server.listen(0, '127.0.0.1', resolveListen)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('missing test port') + await new Promise((resolveClose) => server.close(() => resolveClose())) + return address.port +} + +async function waitForRelay(child: ChildProcess): Promise { + await new Promise((resolveReady, reject) => { + let stderr = '' + const timeout = setTimeout(() => reject(new Error('relay did not start')), 10_000) + child.stdout?.on('data', (chunk: Buffer) => { + if (chunk.toString().includes('[orca-relay] listening')) { + clearTimeout(timeout) + resolveReady() + } + }) + child.stderr?.on('data', (chunk: Buffer) => (stderr += chunk.toString())) + child.once('exit', (code) => + reject(new Error(`relay exited before ready: ${code}\n${stderr}`)) + ) + }) +} + +async function relayToken(audience: string, relayHostId = 'abcdefghijklmnop'): Promise { + return await new SignJWT({ + prof: 'profile-1', + org: 'org-1', + purpose: 'host-control', + relayHostId + }) + .setProtectedHeader({ alg: 'ES256', kid: 'test-key' }) + .setIssuer(issuer) + .setAudience(audience) + .setSubject('user-1') + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey) +} + +async function adminToken(): Promise { + return await googleServiceToken(adminAudience) +} + +async function googleServiceToken(audience: string, email = 'deploy@example.com'): Promise { + return await new SignJWT({ email, email_verified: true }) + .setProtectedHeader({ alg: 'RS256', kid: 'admin-key' }) + .setIssuer('https://accounts.google.com') + .setAudience(audience) + .setSubject('deploy-subject') + .setIssuedAt() + .setExpirationTime('5m') + .sign(adminPrivateKey) +} + +async function postCellHeartbeat( + directorUrl: string, + cell: { id: string; url: string }, + overrides: { incarnation?: string; startedAt?: number; ready?: boolean } = {} +): Promise { + const audience = `${directorUrl}/v1/admin/cell-heartbeat` + return await fetch(audience, { + method: 'POST', + headers: { + authorization: `Bearer ${await googleServiceToken(audience)}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + v: 1, + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: overrides.incarnation ?? '11111111-1111-4111-8111-111111111111', + startedAt: overrides.startedAt ?? Date.now(), + ready: overrides.ready ?? true, + observedRequests: 0 + }) + }) +} + +function spawnTopologyRelay(input: { + url: string + dataDirectory: string + role: 'director' | 'cell' + cellId: string + cells?: Array<{ id: string; url: string; capacityRequests: number }> +}): ChildProcess { + return spawn(process.execPath, ['--import', 'tsx', 'src/index.ts'], { + cwd: appDirectory, + env: { + ...process.env, + PORT: new URL(input.url).port, + ORCA_RELAY_PUBLIC_URL: input.url, + ORCA_RELAY_CELL_URL: input.url, + ORCA_RELAY_CELL_ID: input.cellId, + ORCA_RELAY_CELL_CAPACITY: '10', + ORCA_RELAY_CELLS_JSON: JSON.stringify(input.cells ?? []), + ORCA_RELAY_ROLE: input.role, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: input.dataDirectory, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_MONITOR_SERVICE_ACCOUNT: 'monitor@example.com', + ORCA_RELAY_FENCE_SERVICE_ACCOUNT: 'fence@example.com', + ORCA_RELAY_FENCE_BROKER_SERVICE_ACCOUNT: 'broker@example.com', + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks` + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) +} + +function nextMessage(socket: WebSocket): Promise> { + return new Promise((resolveMessage, reject) => { + const cleanup = (): void => { + socket.off('message', onMessage) + socket.off('error', onError) + socket.off('close', onClose) + } + const onMessage = (data: RawData): void => { + cleanup() + try { + resolveMessage(JSON.parse(data.toString()) as Record) + } catch (error) { + reject(error) + } + } + const onError = (error: Error): void => { + cleanup() + reject(error) + } + const onClose = (code: number, reason: Buffer): void => { + cleanup() + reject(new Error(`socket closed before message: ${code} ${reason.toString()}`)) + } + socket.once('message', onMessage) + socket.once('error', onError) + socket.once('close', onClose) + }) +} + +function nextRawMessage(socket: WebSocket): Promise<{ data: Buffer; binary: boolean }> { + return new Promise((resolveMessage, reject) => { + socket.once('message', (data, binary) => + resolveMessage({ data: Buffer.from(data as ArrayBuffer), binary }) + ) + socket.once('error', reject) + }) +} + +function collectMessages(socket: WebSocket, count: number): Promise[]> { + return new Promise((resolveMessages, reject) => { + const messages: Record[] = [] + const onMessage = (data: RawData): void => { + try { + messages.push(JSON.parse(data.toString()) as Record) + if (messages.length === count) { + socket.off('message', onMessage) + resolveMessages(messages) + } + } catch (error) { + reject(error) + } + } + socket.on('message', onMessage) + socket.once('error', reject) + }) +} + +async function installDirectCredential(input: { + host: WebSocket + relayDeviceId: string + reqId: string + resumeToken: string +}): Promise> { + const response = nextMessage(input.host) + input.host.send( + JSON.stringify({ + type: 'device-credential-install', + v: 1, + reqId: input.reqId, + relayDeviceId: input.relayDeviceId, + newResumeTokenHash: createHash('sha256').update(input.resumeToken).digest('base64url'), + authorization: { mode: 'authenticated-direct', directAuthId: `direct-${input.reqId}` } + }) + ) + return await response +} + +async function attachPhone(input: { + host: WebSocket + hostAck: Record + hostId: string + credential: string +}): Promise<{ + phone: WebSocket + data: WebSocket + connId: string + hello: Record +}> { + const headers = forwardedHeaders() + const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${input.hostId}`, { + headers + }) + await new Promise((resolveOpen, reject) => { + phone.once('open', resolveOpen) + phone.once('error', reject) + }) + const connOpenPromise = nextMessage(input.host) + phone.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: input.credential })) + const connOpen = await connOpenPromise + expect(connOpen.type).toBe('conn-open') + const connId = String(connOpen.connId) + const data = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/data/${connId}`, { + headers + }) + await new Promise((resolveOpen, reject) => { + data.once('open', resolveOpen) + data.once('error', reject) + }) + const helloPromise = nextMessage(phone) + data.send( + JSON.stringify({ + type: 'host-data-auth', + v: 1, + connTicket: connOpen.connTicket, + generation: input.hostAck.generation + }) + ) + const hello = await helloPromise + expect(hello).toMatchObject({ type: 'relay-hello', ok: true }) + return { phone, data, connId, hello } +} + +async function openHostControl(input?: { + controlResumeSecret?: string + previousGeneration?: number + keyPair?: nacl.BoxKeyPair + assignmentEpoch?: number +}): Promise<{ socket: WebSocket; ack: Record; keyPair: nacl.BoxKeyPair }> { + const keyPair = input?.keyPair ?? nacl.box.keyPair() + const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16) + const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, { + headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` }, + perMessageDeflate: false + }) + await new Promise((resolveOpen, reject) => { + socket.once('open', resolveOpen) + socket.once('error', reject) + }) + socket.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: hostId, + assignmentEpoch: input?.assignmentEpoch ?? 1, + hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'), + appVersion: 'test', + ...(input?.controlResumeSecret + ? { controlResumeSecret: input.controlResumeSecret } + : {}), + ...(input?.previousGeneration === undefined + ? {} + : { previousGeneration: input.previousGeneration }) + }) + ) + const challenge = await nextMessage(socket) + expect(challenge.type).toBe('host-challenge') + const plaintext = nacl.box.open( + Buffer.from(String(challenge.ciphertextB64), 'base64'), + Buffer.from(String(challenge.nonceB64), 'base64'), + Buffer.from(String(challenge.relayEphemeralPublicKeyB64), 'base64'), + keyPair.secretKey + ) + if (!plaintext) throw new Error('host challenge did not decrypt') + const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`) + expect(plaintext.slice(0, domain.length)).toEqual(domain) + const transcriptLength = new DataView( + plaintext.buffer, + plaintext.byteOffset + domain.length, + 4 + ).getUint32(0, false) + const transcriptStart = domain.length + 4 + const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength) + const secret = plaintext.slice(transcriptStart + transcriptLength) + const proofB64 = createHmac('sha256', secret) + .update(buildHostProofMacInput(transcript)) + .digest('base64') + socket.send( + JSON.stringify({ + type: 'host-challenge-ack', + challengeId: challenge.challengeId, + proofB64 + }) + ) + return { socket, ack: await nextMessage(socket), keyPair } +} + +beforeAll(async () => { + const keys = await generateKeyPair('ES256') + privateKey = keys.privateKey + const publicJwk = await exportJWK(keys.publicKey) + const adminKeys = await generateKeyPair('RS256') + adminPrivateKey = adminKeys.privateKey + const adminPublicJwk = await exportJWK(adminKeys.publicKey) + jwksServer = createServer((_request, response) => { + response.setHeader('content-type', 'application/json') + response.end( + JSON.stringify({ + keys: [ + { ...publicJwk, kid: 'test-key', alg: 'ES256', use: 'sig' }, + { ...adminPublicJwk, kid: 'admin-key', alg: 'RS256', use: 'sig' } + ] + }) + ) + }) + await new Promise((resolveListen) => jwksServer.listen(0, '127.0.0.1', resolveListen)) + const jwksAddress = jwksServer.address() + if (!jwksAddress || typeof jwksAddress === 'string') throw new Error('missing JWKS address') + issuer = `http://127.0.0.1:${jwksAddress.port}` + const relayPort = await unusedPort() + relayUrl = `http://127.0.0.1:${relayPort}` + adminAudience = `${relayUrl}/v1/admin/drain` + relayDataDirectory = mkdtempSync(resolve(tmpdir(), 'orca-relay-blackbox-')) + relayProcess = spawn(process.execPath, ['--import', 'tsx', 'src/index.ts'], { + cwd: appDirectory, + env: { + ...process.env, + PORT: String(relayPort), + ORCA_RELAY_PUBLIC_URL: relayUrl, + ORCA_RELAY_CELL_URL: relayUrl, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: relayDataDirectory, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_MONITOR_SERVICE_ACCOUNT: 'monitor@example.com', + ORCA_RELAY_FENCE_SERVICE_ACCOUNT: 'fence@example.com', + ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`, + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks` + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + await waitForRelay(relayProcess) +}) + +afterAll(async () => { + relayProcess?.kill('SIGTERM') + await new Promise((resolveClose) => jwksServer?.close(() => resolveClose())) + rmSync(relayDataDirectory, { recursive: true, force: true }) +}) + +describe('served relay URL', () => { + it('exposes only /health, never /healthz', async () => { + expect(await (await fetch(`${relayUrl}/health`)).json()).toEqual({ + ok: true, + connectionCapacityProtocol: 2 + }) + expect(await (await fetch(`${relayUrl}/ready`)).json()).toEqual({ ok: true }) + expect((await fetch(`${relayUrl}/healthz`)).status).toBe(404) + }) + + it('keeps liveness healthy when dependency readiness fails', async () => { + const port = await unusedPort() + const url = `http://127.0.0.1:${port}` + const dataDirectory = mkdtempSync(resolve(tmpdir(), 'orca-relay-unready-')) + const processUnderTest = spawn(process.execPath, ['--import', 'tsx', 'src/index.ts'], { + cwd: appDirectory, + env: { + ...process.env, + PORT: String(port), + ORCA_RELAY_PUBLIC_URL: url, + ORCA_RELAY_CELL_URL: url, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: 'http://127.0.0.1:1/jwks', + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: dataDirectory, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`, + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks` + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + try { + await waitForRelay(processUnderTest) + expect((await fetch(`${url}/health`)).status).toBe(200) + expect((await fetch(`${url}/ready`)).status).toBe(503) + } finally { + processUnderTest.kill('SIGTERM') + await new Promise((resolveExit) => processUnderTest.once('exit', () => resolveExit())) + rmSync(dataDirectory, { recursive: true, force: true }) + } + }) + + it('rejects missing and broad-audience bearer tokens', async () => { + const body = JSON.stringify({ v: 1, relayHostId: 'abcdefghijklmnop' }) + expect((await fetch(`${relayUrl}/v1/assign`, { method: 'POST', body })).status).toBe(401) + expect( + ( + await fetch(`${relayUrl}/v1/assign`, { + method: 'POST', + headers: { authorization: `Bearer ${await relayToken('orca-cloud')}` }, + body + }) + ).status + ).toBe(401) + }) + + it('bounds slow first-frame admission and rejects URL credentials before upgrade', async () => { + const slow: WebSocket[] = [] + for (let index = 0; index < 4; index++) { + const socket = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop` + ) + await new Promise((resolveOpen, reject) => { + socket.once('open', resolveOpen) + socket.once('error', reject) + }) + slow.push(socket) + } + const limited = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop` + ) + const limitedStatus = await new Promise((resolveStatus) => { + limited.once('unexpected-response', (_request, response) => + resolveStatus(response.statusCode ?? 0) + ) + limited.once('error', () => {}) + }) + expect(limitedStatus).toBe(429) + const isolated = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop`, + { headers: { 'x-forwarded-for': 'spoofed, 198.51.100.9, 35.191.0.1' } } + ) + await new Promise((resolveOpen, reject) => { + isolated.once('open', resolveOpen) + isolated.once('error', reject) + }) + isolated.close() + for (const socket of slow) socket.close() + + const leaked = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop?credential=secret` + ) + const leakedStatus = await new Promise((resolveStatus) => { + leaked.once('unexpected-response', (_request, response) => + resolveStatus(response.statusCode ?? 0) + ) + leaked.once('error', () => {}) + }) + expect(leakedStatus).toBe(400) + }) + + it('enforces process-wide slow-auth and per-source rate budgets', async () => { + const slow: WebSocket[] = [] + for (let index = 0; index < 45; index++) { + const socket = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop`, + { headers: { 'x-forwarded-for': `spoofed, 198.51.100.${index + 1}, 35.191.0.1` } } + ) + await new Promise((resolveOpen, reject) => { + socket.once('open', resolveOpen) + socket.once('error', reject) + }) + slow.push(socket) + } + const globallyLimited = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop`, + { headers: { 'x-forwarded-for': 'spoofed, 203.0.113.1, 35.191.0.1' } } + ) + const globalStatus = await new Promise((resolveStatus) => { + globallyLimited.once('unexpected-response', (_request, response) => + resolveStatus(response.statusCode ?? 0) + ) + globallyLimited.once('error', () => {}) + }) + expect(globalStatus).toBe(429) + await Promise.all( + slow.map( + (socket) => + new Promise((resolveClose) => { + socket.once('close', () => resolveClose()) + socket.close() + }) + ) + ) + + for (let index = 0; index < 30; index++) { + const socket = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop`, + { headers: { 'x-forwarded-for': 'spoofed, 203.0.113.2, 35.191.0.1' } } + ) + await new Promise((resolveOpen, reject) => { + socket.once('open', resolveOpen) + socket.once('error', reject) + }) + const closed = new Promise((resolveClose) => socket.once('close', () => resolveClose())) + socket.send('{}') + await closed + } + const rateLimited = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/abcdefghijklmnop`, + { headers: { 'x-forwarded-for': 'spoofed, 203.0.113.2, 35.191.0.1' } } + ) + const rateStatus = await new Promise((resolveStatus) => { + rateLimited.once('unexpected-response', (_request, response) => + resolveStatus(response.statusCode ?? 0) + ) + rateLimited.once('error', () => {}) + }) + expect(rateStatus).toBe(429) + }) + + it('returns a signed combined-service assignment for the bound host', async () => { + const response = await fetch(`${relayUrl}/v1/assign`, { + method: 'POST', + headers: { + authorization: `Bearer ${await relayToken('orca-relay')}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, relayHostId: 'abcdefghijklmnop' }) + }) + expect(response.status).toBe(200) + const assignment = (await response.json()) as { + v: number + cellUrl: string + assignmentEpoch: number + lease: string + } + expect(assignment).toMatchObject({ v: 1, cellUrl: relayUrl, assignmentEpoch: 1 }) + const verified = await jwtVerify(assignment.lease, new TextEncoder().encode(assignmentKey), { + issuer: relayUrl, + audience: 'orca-relay-cell', + algorithms: ['HS256'] + }) + expect(verified.payload).toMatchObject({ relayHostId: 'abcdefghijklmnop' }) + }) + + it('requires canonical host key possession and supports same-generation rebind', async () => { + const first = await openHostControl() + expect(first.ack).toMatchObject({ type: 'host-hello-ack', v: 1, generation: 1 }) + const firstClose = new Promise((resolveClose) => + first.socket.once('close', (code) => resolveClose(code)) + ) + const rebound = await openHostControl({ + keyPair: first.keyPair, + controlResumeSecret: String(first.ack.controlResumeSecret), + previousGeneration: 1 + }) + expect(rebound.ack).toMatchObject({ type: 'host-hello-ack', v: 1, generation: 1 }) + expect(await firstClose).toBe(4408) + rebound.socket.close() + }) + + it('rejects a scoped token presented by a different host key', async () => { + const claimedKey = nacl.box.keyPair() + const wrongKey = nacl.box.keyPair() + const hostId = createHash('sha256').update(claimedKey.publicKey).digest('base64url').slice(0, 16) + const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, { + headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` } + }) + await new Promise((resolveOpen) => socket.once('open', resolveOpen)) + const closed = new Promise((resolveClose) => + socket.once('close', (code) => resolveClose(code)) + ) + socket.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: hostId, + assignmentEpoch: 1, + hostPublicKeyB64: Buffer.from(wrongKey.publicKey).toString('base64'), + appVersion: 'test' + }) + ) + expect(await closed).toBe(4401) + }) + + it('returns typed 4409 without a cell URL for a stale assignment epoch', async () => { + const keyPair = nacl.box.keyPair() + const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16) + const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, { + headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` } + }) + await new Promise((resolveOpen, reject) => { + socket.once('open', resolveOpen) + socket.once('error', reject) + }) + const closed = new Promise<{ code: number; reason: string }>((resolveClose) => + socket.once('close', (code, reason) => + resolveClose({ code, reason: reason.toString() }) + ) + ) + socket.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: hostId, + assignmentEpoch: 2, + hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'), + appVersion: 'test' + }) + ) + const result = await closed + expect(result.code).toBe(4409) + expect(result.reason).not.toContain('http') + }) + + it('keeps a pending attach usable after a bad ticket and rejects ticket replay', async () => { + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const inviteResponse = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ type: 'invite-create', reqId: 'ticket-invite', relayDeviceId: 'ticket-device' }) + ) + const invite = await inviteResponse + const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, { + headers: forwardedHeaders() + }) + await new Promise((resolveOpen, reject) => { + phone.once('open', resolveOpen) + phone.once('error', reject) + }) + const connectionPromise = nextMessage(host.socket) + phone.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) + ) + const connection = await connectionPromise + const dataUrl = `${relayUrl.replace('http:', 'ws:')}/v1/host/data/${connection.connId}` + + const badData = new WebSocket(dataUrl, { headers: forwardedHeaders() }) + await new Promise((resolveOpen, reject) => { + badData.once('open', resolveOpen) + badData.once('error', reject) + }) + const badClosed = new Promise((resolveClose) => + badData.once('close', (code) => resolveClose(code)) + ) + badData.send( + JSON.stringify({ + type: 'host-data-auth', + v: 1, + connTicket: Buffer.alloc(32, 1).toString('base64url'), + generation: host.ack.generation + }) + ) + expect(await badClosed).toBe(4401) + + const data = new WebSocket(dataUrl, { headers: forwardedHeaders() }) + await new Promise((resolveOpen, reject) => { + data.once('open', resolveOpen) + data.once('error', reject) + }) + const hello = nextMessage(phone) + data.send( + JSON.stringify({ + type: 'host-data-auth', + v: 1, + connTicket: connection.connTicket, + generation: host.ack.generation + }) + ) + expect(await hello).toMatchObject({ type: 'relay-hello', ok: true }) + + const replay = new WebSocket(dataUrl, { headers: forwardedHeaders() }) + await new Promise((resolveOpen, reject) => { + replay.once('open', resolveOpen) + replay.once('error', reject) + }) + const replayClosed = new Promise((resolveClose) => + replay.once('close', (code) => resolveClose(code)) + ) + replay.send( + JSON.stringify({ + type: 'host-data-auth', + v: 1, + connTicket: connection.connTicket, + generation: host.ack.generation + }) + ) + expect(await replayClosed).toBe(4401) + phone.close() + data.close() + host.socket.close() + }) + + it('attaches before success, preserves text/binary opcodes, installs, and confirms resume', async () => { + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const invitePromise = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ type: 'invite-create', reqId: 'invite-1', relayDeviceId: 'device-1' }) + ) + const invite = await invitePromise + expect(invite.type).toBe('invite-created') + const first = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: String(invite.inviteToken) + }) + + const hostText = nextRawMessage(first.data) + first.phone.send('phone-text') + expect(await hostText).toEqual({ data: Buffer.from('phone-text'), binary: false }) + const phoneBinary = nextRawMessage(first.phone) + first.data.send(Buffer.from([1, 2, 3]), { binary: true }) + expect(await phoneBinary).toEqual({ data: Buffer.from([1, 2, 3]), binary: true }) + + const resumeToken = Buffer.alloc(32, 9).toString('base64url') + const installedPromise = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-credential-install', + v: 1, + reqId: 'install-1', + relayDeviceId: 'device-1', + newResumeTokenHash: createHash('sha256').update(resumeToken).digest('base64url'), + authorization: { mode: 'relay-basis', basisConnId: first.connId } + }) + ) + const installed = await installedPromise + expect(installed).toMatchObject({ + type: 'device-credential-installed', + authorizationMode: 'relay-basis', + currentVersion: 1 + }) + const resolved = await fetch(`${relayUrl}/v1/resolve`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, relayHostId: hostId, resumeToken }) + }) + expect(resolved.status).toBe(200) + expect(await resolved.json()).toMatchObject({ v: 1, cellUrl: relayUrl, assignmentEpoch: 1 }) + first.phone.close() + + const resumed = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: resumeToken + }) + const confirmedPromise = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'confirm-1', + basisConnId: resumed.connId + }) + ) + expect(await confirmedPromise).toMatchObject({ + type: 'device-resume-confirmed', + renewed: true, + acceptedAs: 'current' + }) + resumed.phone.close() + resumed.data.close() + host.socket.close() + }) + + it('does not renew outer-only or injected confirmations and reports offline/peer loss', async () => { + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const resumeToken = Buffer.alloc(32, 11).toString('base64url') + expect( + await installDirectCredential({ + host: host.socket, + relayDeviceId: 'outer-device', + reqId: 'outer-install', + resumeToken + }) + ).toMatchObject({ type: 'device-credential-installed', currentVersion: 1 }) + + const first = await attachPhone({ host: host.socket, hostAck: host.ack, hostId, credential: resumeToken }) + const originalExpiry = first.hello.resumeExpiresAt + first.phone.close() + first.data.close() + await new Promise((resolveWait) => setTimeout(resolveWait, 25)) + const closedBasis = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'closed-basis-confirm', + basisConnId: first.connId + }) + ) + expect(await closedBasis).toMatchObject({ + type: 'control-error', + reqId: 'closed-basis-confirm', + code: 'confirmation_not_active' + }) + const second = await attachPhone({ host: host.socket, hostAck: host.ack, hostId, credential: resumeToken }) + expect(second.hello.resumeExpiresAt).toBe(originalExpiry) + const injectedResult = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'injected-confirm', + basisConnId: second.connId, + relayDeviceId: 'injected', + acceptedCredentialVersion: 99 + }) + ) + expect(await injectedResult).toMatchObject({ type: 'control-error', reqId: 'injected-confirm' }) + const phoneClosed = new Promise((resolveClose) => + second.phone.once('close', (code) => resolveClose(code)) + ) + second.data.close() + expect(await phoneClosed).toBe(4408) + + await new Promise((resolveClose) => { + host.socket.once('close', () => resolveClose()) + host.socket.close() + }) + const offline = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, { + headers: forwardedHeaders() + }) + await new Promise((resolveOpen) => offline.once('open', resolveOpen)) + const offlineHello = nextMessage(offline) + offline.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: resumeToken }) + ) + expect(await offlineHello).toEqual({ type: 'relay-hello', ok: false, code: 4404 }) + }) + + it('rejects the first post-E2EE confirmation after its server-owned deadline', async () => { + const originalUrl = relayUrl + const clockPort = await unusedPort() + const clockUrl = `http://127.0.0.1:${clockPort}` + const clockData = mkdtempSync(resolve(tmpdir(), 'orca-relay-clock-')) + const clockFile = resolve(clockData, 'offset-ms') + writeFileSync(clockFile, '0') + const clockProcess = spawn( + process.execPath, + ['--import', 'tsx', 'src/fault-injection-test-entry.ts'], + { + cwd: appDirectory, + env: { + ...process.env, + NODE_ENV: 'test', + PORT: String(clockPort), + ORCA_RELAY_PUBLIC_URL: clockUrl, + ORCA_RELAY_CELL_URL: clockUrl, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: clockData, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_TEST_CLOCK_FILE: clockFile + }, + stdio: ['ignore', 'pipe', 'pipe'] + } + ) + relayUrl = clockUrl + try { + await waitForRelay(clockProcess) + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const resumeToken = Buffer.alloc(32, 25).toString('base64url') + await installDirectCredential({ + host: host.socket, + relayDeviceId: 'late-confirm-device', + reqId: 'late-confirm-install', + resumeToken + }) + const splice = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: resumeToken + }) + writeFileSync(clockFile, '31000') + const rejected = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'late-first-confirm', + basisConnId: splice.connId + }) + ) + expect(await rejected).toMatchObject({ + type: 'control-error', + reqId: 'late-first-confirm', + code: 'confirmation_not_active' + }) + splice.phone.close() + splice.data.close() + host.socket.close() + } finally { + clockProcess.kill('SIGKILL') + await new Promise((resolveExit) => clockProcess.once('exit', () => resolveExit())) + relayUrl = originalUrl + rmSync(clockData, { recursive: true, force: true }) + } + }) + + it('fences a competing generation and rejects its old immutable basis', async () => { + const firstHost = await openHostControl() + const hostId = createHash('sha256') + .update(firstHost.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const resumeToken = Buffer.alloc(32, 12).toString('base64url') + await installDirectCredential({ + host: firstHost.socket, + relayDeviceId: 'fenced-device', + reqId: 'fenced-install', + resumeToken + }) + const splice = await attachPhone({ + host: firstHost.socket, + hostAck: firstHost.ack, + hostId, + credential: resumeToken + }) + const replacement = await openHostControl({ keyPair: firstHost.keyPair, previousGeneration: 1 }) + expect(replacement.ack.generation).toBe(2) + const rejected = nextMessage(replacement.socket) + replacement.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'wrong-generation-confirm', + basisConnId: splice.connId + }) + ) + expect(await rejected).toMatchObject({ + type: 'control-error', + reqId: 'wrong-generation-confirm', + code: 'confirmation_not_active' + }) + replacement.socket.close() + }) + + it('serializes late direct and invite authorization modes into one served result', async () => { + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const inviteResponse = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ type: 'invite-create', reqId: 'race-invite', relayDeviceId: 'race-device' }) + ) + const invite = await inviteResponse + const splice = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: String(invite.inviteToken) + }) + const responses = collectMessages(host.socket, 2) + const base = { + type: 'device-credential-install', + v: 1, + reqId: 'race-install', + relayDeviceId: 'race-device', + newResumeTokenHash: createHash('sha256').update('race-resume').digest('base64url') + } + host.socket.send( + JSON.stringify({ + ...base, + authorization: { mode: 'relay-basis', basisConnId: splice.connId } + }) + ) + host.socket.send( + JSON.stringify({ + ...base, + authorization: { mode: 'authenticated-direct', directAuthId: 'late-direct' } + }) + ) + const installed = await responses + expect(installed).toHaveLength(2) + expect(installed[0]).toEqual(installed[1]) + expect(installed[0]).toMatchObject({ type: 'device-credential-installed', currentVersion: 1 }) + splice.phone.close() + splice.data.close() + host.socket.close() + }) + + it('reconciles a direct commit after its response is ignored and rejects bad authorization', async () => { + const firstHost = await openHostControl() + const hostId = createHash('sha256') + .update(firstHost.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const inviteResponse = nextMessage(firstHost.socket) + firstHost.socket.send( + JSON.stringify({ + type: 'invite-create', + reqId: 'lost-response-invite', + relayDeviceId: 'lost-response-device' + }) + ) + const invite = await inviteResponse + const resumeToken = Buffer.alloc(32, 26).toString('base64url') + firstHost.socket.send( + JSON.stringify({ + type: 'device-credential-install', + v: 1, + reqId: 'lost-response-install', + relayDeviceId: 'lost-response-device', + newResumeTokenHash: createHash('sha256').update(resumeToken).digest('base64url'), + authorization: { mode: 'authenticated-direct', directAuthId: 'lost-response-direct' } + }) + ) + // The coordinator deliberately ignores the acknowledgement and recovers + // only from the durable status after its control transport disappears. + await new Promise((resolveWait) => setTimeout(resolveWait, 25)) + firstHost.socket.terminate() + const rebound = await openHostControl({ + keyPair: firstHost.keyPair, + controlResumeSecret: String(firstHost.ack.controlResumeSecret), + previousGeneration: Number(firstHost.ack.generation) + }) + const status = nextMessage(rebound.socket) + rebound.socket.send( + JSON.stringify({ + type: 'device-credential-install-status', + v: 1, + reqId: 'lost-response-install', + relayDeviceId: 'lost-response-device' + }) + ) + expect(await status).toMatchObject({ + type: 'device-credential-install-status-result', + state: 'committed', + result: { authorizationMode: 'authenticated-direct', currentVersion: 1 } + }) + + const invalidatedInvite = new WebSocket( + `${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, + { headers: forwardedHeaders() } + ) + await new Promise((resolveOpen, reject) => { + invalidatedInvite.once('open', resolveOpen) + invalidatedInvite.once('error', reject) + }) + const rejectedInvite = nextMessage(invalidatedInvite) + invalidatedInvite.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) + ) + expect(await rejectedInvite).toEqual({ type: 'relay-hello', ok: false, code: 4401 }) + + const unauthorized = nextMessage(rebound.socket) + rebound.socket.send( + JSON.stringify({ + type: 'device-credential-install', + v: 1, + reqId: 'unauthorized-install', + relayDeviceId: 'unauthorized-device', + newResumeTokenHash: createHash('sha256').update('unauthorized').digest('base64url'), + authorization: { mode: 'relay-basis', basisConnId: 'unknown-basis' } + }) + ) + expect(await unauthorized).toMatchObject({ + type: 'control-error', + reqId: 'unauthorized-install', + code: 'invalid_relay_basis' + }) + const missing = nextMessage(rebound.socket) + rebound.socket.send( + JSON.stringify({ + type: 'device-credential-install-status', + v: 1, + reqId: 'unauthorized-install', + relayDeviceId: 'unauthorized-device' + }) + ) + expect(await missing).toMatchObject({ + type: 'device-credential-install-status-result', + state: 'not-found' + }) + rebound.socket.close() + }) + + it('serializes confirmation against direct rotation, relay rotation, and revoke', async () => { + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const firstToken = Buffer.alloc(32, 21).toString('base64url') + await installDirectCredential({ + host: host.socket, + relayDeviceId: 'confirm-race-device', + reqId: 'confirm-race-initial', + resumeToken: firstToken + }) + + const firstResume = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: firstToken + }) + const secondToken = Buffer.alloc(32, 22).toString('base64url') + const directResponses = collectMessages(host.socket, 2) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'confirm-before-direct', + basisConnId: firstResume.connId + }) + ) + host.socket.send( + JSON.stringify({ + type: 'device-credential-install', + v: 1, + reqId: 'direct-after-confirm', + relayDeviceId: 'confirm-race-device', + newResumeTokenHash: createHash('sha256').update(secondToken).digest('base64url'), + expectedCurrentHash: createHash('sha256').update(firstToken).digest('base64url'), + authorization: { mode: 'authenticated-direct', directAuthId: 'direct-after-confirm' } + }) + ) + const directResults = await directResponses + expect(directResults.find((result) => result.type === 'device-resume-confirmed')).toMatchObject({ + reqId: 'confirm-before-direct', + currentVersion: 1, + renewed: true + }) + expect(directResults.find((result) => result.type === 'device-credential-installed')).toMatchObject({ + reqId: 'direct-after-confirm', + currentVersion: 2 + }) + const replayPromise = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'confirm-before-direct', + basisConnId: firstResume.connId + }) + ) + expect(await replayPromise).toEqual( + directResults.find((result) => result.type === 'device-resume-confirmed') + ) + + const secondResume = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: secondToken + }) + const inviteResponse = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'invite-create', + reqId: 'relay-rotation-invite', + relayDeviceId: 'confirm-race-device' + }) + ) + const invite = await inviteResponse + const inviteSplice = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: String(invite.inviteToken) + }) + const thirdToken = Buffer.alloc(32, 23).toString('base64url') + const relayResponses = collectMessages(host.socket, 2) + host.socket.send( + JSON.stringify({ + type: 'device-credential-install', + v: 1, + reqId: 'relay-before-confirm', + relayDeviceId: 'confirm-race-device', + newResumeTokenHash: createHash('sha256').update(thirdToken).digest('base64url'), + expectedCurrentHash: createHash('sha256').update(secondToken).digest('base64url'), + authorization: { mode: 'relay-basis', basisConnId: inviteSplice.connId } + }) + ) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'confirm-after-relay', + basisConnId: secondResume.connId + }) + ) + const relayResults = await relayResponses + expect(relayResults.find((result) => result.type === 'device-credential-installed')).toMatchObject({ + reqId: 'relay-before-confirm', + currentVersion: 3 + }) + expect(relayResults.find((result) => result.type === 'device-resume-confirmed')).toMatchObject({ + reqId: 'confirm-after-relay', + currentVersion: 3, + renewed: false + }) + + const retiredResume = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: secondToken + }) + const fourthToken = Buffer.alloc(32, 24).toString('base64url') + expect( + await installDirectCredential({ + host: host.socket, + relayDeviceId: 'confirm-race-device', + reqId: 'retire-before-confirm', + resumeToken: fourthToken + }) + ).toMatchObject({ type: 'device-credential-installed', currentVersion: 4 }) + const retiredResult = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'confirm-retired', + basisConnId: retiredResume.connId + }) + ) + expect(await retiredResult).toMatchObject({ + type: 'control-error', + reqId: 'confirm-retired', + code: 'reject-retired' + }) + + const currentResume = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: fourthToken + }) + const revokeResponses = collectMessages(host.socket, 2) + host.socket.send( + JSON.stringify({ + type: 'device-revoke', + reqId: 'revoke-before-confirm', + relayDeviceId: 'confirm-race-device' + }) + ) + host.socket.send( + JSON.stringify({ + type: 'device-resume-confirm', + v: 1, + reqId: 'confirm-after-revoke', + basisConnId: currentResume.connId + }) + ) + const revokeResults = await revokeResponses + expect(revokeResults.find((result) => result.type === 'device-revoked')).toMatchObject({ + reqId: 'revoke-before-confirm' + }) + expect(revokeResults.find((result) => result.type === 'control-error')).toMatchObject({ + reqId: 'confirm-after-revoke', + code: 'reject-revoked' + }) + + for (const splice of [firstResume, secondResume, inviteSplice, retiredResume, currentResume]) { + splice.phone.close() + splice.data.close() + } + host.socket.close() + }) + + it('enforces the eight-splice host limit with typed 4429 recovery', async () => { + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const resumeToken = Buffer.alloc(32, 13).toString('base64url') + await installDirectCredential({ + host: host.socket, + relayDeviceId: 'limit-device', + reqId: 'limit-install', + resumeToken + }) + const splices = [] + for (let index = 0; index < 8; index++) { + splices.push( + await attachPhone({ host: host.socket, hostAck: host.ack, hostId, credential: resumeToken }) + ) + } + const ninth = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, { + headers: forwardedHeaders() + }) + await new Promise((resolveOpen) => ninth.once('open', resolveOpen)) + const rejected = nextMessage(ninth) + ninth.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: resumeToken }) + ) + expect(await rejected).toEqual({ type: 'relay-hello', ok: false, code: 4429 }) + for (const splice of splices) { + splice.phone.close() + splice.data.close() + } + host.socket.close() + }) + + it('rolls an aborted invite reservation into bounded cooldown before retry', async () => { + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const inviteResponse = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ type: 'invite-create', reqId: 'cooldown-invite', relayDeviceId: 'cooldown-device' }) + ) + const invite = await inviteResponse + const first = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, { + headers: forwardedHeaders() + }) + await new Promise((resolveOpen) => first.once('open', resolveOpen)) + const firstOpen = nextMessage(host.socket) + first.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) + ) + await firstOpen + first.close() + await new Promise((resolveWait) => setTimeout(resolveWait, 50)) + + const cooldown = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, { + headers: forwardedHeaders() + }) + await new Promise((resolveOpen) => cooldown.once('open', resolveOpen)) + const cooldownHello = nextMessage(cooldown) + cooldown.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) + ) + expect(await cooldownHello).toEqual({ type: 'relay-hello', ok: false, code: 4401 }) + + await new Promise((resolveWait) => setTimeout(resolveWait, 2_050)) + const retry = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, { + headers: forwardedHeaders() + }) + await new Promise((resolveOpen) => retry.once('open', resolveOpen)) + const retriedOpen = nextMessage(host.socket) + retry.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) + ) + expect(await retriedOpen).toMatchObject({ type: 'conn-open', kind: 'invite' }) + retry.close() + host.socket.close() + }) + + it('keeps install status and every effect not-found after injected SQL failure', async () => { + const originalUrl = relayUrl + const faultPort = await unusedPort() + const faultUrl = `http://127.0.0.1:${faultPort}` + const faultData = mkdtempSync(resolve(tmpdir(), 'orca-relay-fault-')) + const faultProcess = spawn( + process.execPath, + ['--import', 'tsx', 'src/fault-injection-test-entry.ts'], + { + cwd: appDirectory, + env: { + ...process.env, + NODE_ENV: 'test', + PORT: String(faultPort), + ORCA_RELAY_PUBLIC_URL: faultUrl, + ORCA_RELAY_CELL_URL: faultUrl, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: faultData, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_TEST_FAULT_SQL: 'INSERT INTO relay_install_results' + }, + stdio: ['ignore', 'pipe', 'pipe'] + } + ) + relayUrl = faultUrl + try { + await waitForRelay(faultProcess) + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const inviteResponse = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ type: 'invite-create', reqId: 'fault-invite', relayDeviceId: 'fault-device' }) + ) + const invite = await inviteResponse + const splice = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: String(invite.inviteToken) + }) + const install = { + type: 'device-credential-install', + v: 1, + reqId: 'fault-install', + relayDeviceId: 'fault-device', + newResumeTokenHash: createHash('sha256').update('fault-resume').digest('base64url'), + authorization: { mode: 'relay-basis', basisConnId: splice.connId } + } + const failed = nextMessage(host.socket) + host.socket.send(JSON.stringify(install)) + expect(await failed).toMatchObject({ + type: 'control-error', + reqId: 'fault-install', + code: 'injected SQL failure' + }) + const status = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-credential-install-status', + v: 1, + reqId: 'fault-install', + relayDeviceId: 'fault-device' + }) + ) + expect(await status).toMatchObject({ + type: 'device-credential-install-status-result', + state: 'not-found' + }) + const retried = nextMessage(host.socket) + host.socket.send(JSON.stringify(install)) + expect(await retried).toMatchObject({ + type: 'device-credential-installed', + currentVersion: 1 + }) + splice.phone.close() + splice.data.close() + host.socket.close() + } finally { + faultProcess.kill('SIGKILL') + await new Promise((resolveExit) => faultProcess.once('exit', () => resolveExit())) + relayUrl = originalUrl + rmSync(faultData, { recursive: true, force: true }) + } + }) + + it('falls back to an invite only after a failed direct attempt is authoritatively not-found', async () => { + const originalUrl = relayUrl + const faultPort = await unusedPort() + const faultUrl = `http://127.0.0.1:${faultPort}` + const faultData = mkdtempSync(resolve(tmpdir(), 'orca-relay-direct-fault-')) + const faultProcess = spawn( + process.execPath, + ['--import', 'tsx', 'src/fault-injection-test-entry.ts'], + { + cwd: appDirectory, + env: { + ...process.env, + NODE_ENV: 'test', + PORT: String(faultPort), + ORCA_RELAY_PUBLIC_URL: faultUrl, + ORCA_RELAY_CELL_URL: faultUrl, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: faultData, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_TEST_FAULT_SQL: 'INSERT INTO relay_direct_authorizations' + }, + stdio: ['ignore', 'pipe', 'pipe'] + } + ) + relayUrl = faultUrl + try { + await waitForRelay(faultProcess) + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const inviteResponse = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'invite-create', + reqId: 'fallback-invite', + relayDeviceId: 'fallback-device' + }) + ) + const invite = await inviteResponse + const splice = await attachPhone({ + host: host.socket, + hostAck: host.ack, + hostId, + credential: String(invite.inviteToken) + }) + const installBase = { + type: 'device-credential-install', + v: 1, + reqId: 'fallback-install', + relayDeviceId: 'fallback-device', + newResumeTokenHash: createHash('sha256').update('fallback-resume').digest('base64url') + } + const directFailure = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + ...installBase, + authorization: { mode: 'authenticated-direct', directAuthId: 'failed-direct' } + }) + ) + expect(await directFailure).toMatchObject({ + type: 'control-error', + reqId: 'fallback-install', + code: 'injected SQL failure' + }) + const status = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + type: 'device-credential-install-status', + v: 1, + reqId: 'fallback-install', + relayDeviceId: 'fallback-device' + }) + ) + expect(await status).toMatchObject({ + type: 'device-credential-install-status-result', + state: 'not-found' + }) + const fallback = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ + ...installBase, + authorization: { mode: 'relay-basis', basisConnId: splice.connId } + }) + ) + expect(await fallback).toMatchObject({ + type: 'device-credential-installed', + authorizationMode: 'relay-basis', + currentVersion: 1 + }) + splice.phone.close() + splice.data.close() + host.socket.close() + } finally { + faultProcess.kill('SIGKILL') + await new Promise((resolveExit) => faultProcess.once('exit', () => resolveExit())) + relayUrl = originalUrl + rmSync(faultData, { recursive: true, force: true }) + } + }) + + it('keeps director HTTP routes off cells and enforces the durable cell epoch', async () => { + const originalUrl = relayUrl + const cellPort = await unusedPort() + const cellUrl = `http://127.0.0.1:${cellPort}` + const cellData = mkdtempSync(resolve(tmpdir(), 'orca-relay-cell-')) + const keyPair = nacl.box.keyPair() + const hostId = createHash('sha256') + .update(keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const database = await openRelayDatabase({ dataDir: cellData }) + const assignments = new RelayAssignmentStore(database, () => 100) + await assignments.reconcileCells([{ id: 'cell-a', url: cellUrl, capacityRequests: 10 }]) + await assignments.assign({ userId: 'user-1', relayHostId: hostId }) + await database.close() + const cellProcess = spawn(process.execPath, ['--import', 'tsx', 'src/index.ts'], { + cwd: appDirectory, + env: { + ...process.env, + PORT: String(cellPort), + ORCA_RELAY_PUBLIC_URL: cellUrl, + ORCA_RELAY_CELL_URL: cellUrl, + ORCA_RELAY_CELL_ID: 'cell-a', + ORCA_RELAY_CELL_CAPACITY: '10', + ORCA_RELAY_ROLE: 'cell', + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: cellData, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks` + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + relayUrl = cellUrl + try { + await waitForRelay(cellProcess) + const token = await relayToken('orca-relay', hostId) + const assignmentResponse = await fetch(`${cellUrl}/v1/assign`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, relayHostId: hostId }) + }) + expect(assignmentResponse.status).toBe(404) + const cellStatusResponse = await fetch(`${cellUrl}/v1/admin/cell-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId: 'cell-a' }) + }) + expect(cellStatusResponse.status).toBe(404) + + const stale = new WebSocket(`${cellUrl.replace('http:', 'ws:')}/v1/host/control`, { + headers: { authorization: `Bearer ${token}` } + }) + await new Promise((resolveOpen) => stale.once('open', resolveOpen)) + const staleClosed = new Promise((resolveClose) => + stale.once('close', (code) => resolveClose(code)) + ) + stale.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: hostId, + assignmentEpoch: 2, + hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'), + appVersion: 'test' + }) + ) + expect(await staleClosed).toBe(4409) + + const assigned = await openHostControl({ keyPair }) + expect(assigned.ack).toMatchObject({ type: 'host-hello-ack', generation: 1 }) + const invitePromise = nextMessage(assigned.socket) + assigned.socket.send( + JSON.stringify({ type: 'invite-create', reqId: 'cell-invite', relayDeviceId: 'cell-device' }) + ) + const invite = await invitePromise + const splice = await attachPhone({ + host: assigned.socket, + hostAck: assigned.ack, + hostId, + credential: String(invite.inviteToken) + }) + const observedDatabase = await openRelayDatabase({ dataDir: cellData }) + const activities = await observedDatabase.query( + `SELECT activity_kind, request_units FROM relay_assignment_activity_leases + ORDER BY activity_kind, activity_id` + ) + await observedDatabase.close() + expect(activities).toEqual([ + { activity_kind: 'control', request_units: 1 }, + { activity_kind: 'invite', request_units: 1 }, + { activity_kind: 'invite', request_units: 1 }, + { activity_kind: 'splice', request_units: 2 } + ]) + splice.phone.close() + splice.data.close() + assigned.socket.close() + } finally { + cellProcess.kill('SIGTERM') + await new Promise((resolveExit) => cellProcess.once('exit', () => resolveExit())) + relayUrl = originalUrl + rmSync(cellData, { recursive: true, force: true }) + } + }) + + it('runs target-first dual-cell evacuation before releasing the source control', async () => { + const originalUrl = relayUrl + const dataDirectory = mkdtempSync(resolve(tmpdir(), 'orca-relay-topology-')) + const directorUrl = `http://127.0.0.1:${await unusedPort()}` + const cellAUrl = `http://127.0.0.1:${await unusedPort()}` + const cellBUrl = `http://127.0.0.1:${await unusedPort()}` + const cells = [ + { id: 'cell-a', url: cellAUrl, capacityRequests: 10 }, + { id: 'cell-b', url: cellBUrl, capacityRequests: 10 } + ] + const keyPair = nacl.box.keyPair() + const hostId = createHash('sha256') + .update(keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const seedDatabase = await openRelayDatabase({ dataDir: dataDirectory }) + const seedAssignments = new RelayAssignmentStore(seedDatabase) + await seedAssignments.reconcileCells(cells) + await seedAssignments.assign({ userId: 'user-1', relayHostId: hostId }) + await seedDatabase.close() + + const cellA = spawnTopologyRelay({ + url: cellAUrl, + dataDirectory, + role: 'cell', + cellId: 'cell-a' + }) + let director: ChildProcess | undefined + let cellB: ChildProcess | undefined + try { + await waitForRelay(cellA) + director = spawnTopologyRelay({ + url: directorUrl, + dataDirectory, + role: 'director', + cellId: 'director', + cells + }) + await waitForRelay(director) + expect( + await fetch(`${directorUrl}/v1/admin/cell-heartbeat`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{}' + }) + ).toMatchObject({ status: 401 }) + const heartbeatAudience = `${directorUrl}/v1/admin/cell-heartbeat` + expect( + await fetch(heartbeatAudience, { + method: 'POST', + headers: { + authorization: `Bearer ${await googleServiceToken(`${directorUrl}/wrong`)}`, + 'content-type': 'application/json' + }, + body: '{}' + }) + ).toMatchObject({ status: 401 }) + expect( + await fetch(heartbeatAudience, { + method: 'POST', + headers: { + authorization: `Bearer ${await googleServiceToken(heartbeatAudience, 'wrong@example.com')}`, + 'content-type': 'application/json' + }, + body: '{}' + }) + ).toMatchObject({ status: 401 }) + expect(await postCellHeartbeat(directorUrl, cells[0]!, { startedAt: 1_000 })).toMatchObject({ + status: 200 + }) + expect( + await postCellHeartbeat(directorUrl, cells[0]!, { + incarnation: '33333333-3333-4333-8333-333333333333', + startedAt: 999 + }) + ).toMatchObject({ status: 409 }) + expect( + await postCellHeartbeat(directorUrl, cells[1]!, { + incarnation: '22222222-2222-4222-8222-222222222222', + startedAt: 2_000 + }) + ).toMatchObject({ status: 200 }) + relayUrl = cellAUrl + const sourceHost = await openHostControl({ keyPair, assignmentEpoch: 1 }) + const token = await adminToken() + const recoveryRequests = [ + { + path: 'cell-fence-attempt-prepare', + body: { + v: 1, + attemptId: '44444444-4444-4444-8444-444444444444', + environment: 'production', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: + 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + planObjectName: + 'terraform/state/relay-fence-plans/production/44444444-4444-4444-8444-444444444444.tfplan', + varFileSha256: 'c'.repeat(64), + terraformStateLineage: 'c739dab4-e6e1-e627-02a9-504b3dda1a2c', + terraformStateSerial: 7, + terraformStateObjectGeneration: '987654321', + terraformStateObjectSha256: 'd'.repeat(64), + requestReason: + 'orca-relay-fence/44444444-4444-4444-8444-444444444444' + }, + confirmation: 'PREPARE_TERRAFORM_CELL_FENCE', + expected: { status: 409, body: { error: 'cell_fence_admission_enabled' } } + }, + { + path: 'cell-fence-attempt-abort', + body: { + v: 1, + attemptId: '44444444-4444-4444-8444-444444444444', + environment: 'production', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: 'orca-relay-c1', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: + 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenceCommit: 'a'.repeat(40), + planSha256: 'b'.repeat(64), + planObjectName: + 'terraform/state/relay-fence-plans/production/44444444-4444-4444-8444-444444444444.tfplan', + varFileSha256: 'c'.repeat(64), + terraformStateLineage: 'c739dab4-e6e1-e627-02a9-504b3dda1a2c', + terraformStateSerial: 7, + terraformStateObjectGeneration: '987654321', + terraformStateObjectSha256: 'd'.repeat(64), + requestReason: + 'orca-relay-fence/44444444-4444-4444-8444-444444444444' + }, + confirmation: 'ABORT_UNSTARTED_TERRAFORM_CELL_FENCE', + expected: { status: 409, body: { error: 'cell_fence_attempt_not_found' } } + }, + { + path: 'migration-supersede-cell', + body: { + v: 1, + sourceCellId: 'cell-a', + currentTargetCellId: 'cell-b', + replacementTargetCellId: 'cell-c', + limit: 100 + }, + confirmation: 'SUPERSEDE_REGISTERED_CELL_MIGRATIONS', + expected: { status: 200, body: { v: 1, superseded: 0 } } + }, + { + path: 'drain-attempt-prepare', + body: { + v: 1, + attemptId: '55555555-5555-4555-8555-555555555555', + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: '66666666-6666-4666-8666-666666666666', + graceMs: 120_000 + }, + confirmation: 'PREPARE_LEGACY_DRAIN', + expected: { status: 409, body: { error: 'drain_attempt_admission_enabled' } } + }, + { + path: 'drain-attempt-recover-forward', + body: { + v: 1, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111' + }, + confirmation: 'RECOVER_LEGACY_DRAIN', + expected: { status: 409, body: { error: 'drain_attempt_admission_enabled' } } + } + ] + for (const request of recoveryRequests) { + const url = `${directorUrl}/v1/admin/${request.path}` + expect( + await fetch(url, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(request.body) + }) + ).toMatchObject({ status: 400 }) + const confirmed = await fetch(url, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ ...request.body, confirmation: request.confirmation }) + }) + expect(confirmed.status).toBe(request.expected.status) + expect(await confirmed.json()).toEqual(request.expected.body) + } + const statusUrl = `${directorUrl}/v1/admin/cell-status` + expect( + await fetch(statusUrl, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId: 'cell-a' }) + }) + ).toMatchObject({ status: 401 }) + expect( + await fetch(statusUrl, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId: 'cell-a', userId: 'must-not-be-accepted' }) + }) + ).toMatchObject({ status: 400 }) + const sourceStatusResponse = await fetch(statusUrl, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId: 'cell-a' }) + }) + expect(sourceStatusResponse.status).toBe(200) + const sourceStatusText = await sourceStatusResponse.text() + expect(sourceStatusText).not.toContain('user-1') + expect(sourceStatusText).not.toContain(hostId) + expect(JSON.parse(sourceStatusText)).toMatchObject({ + v: 1, + status: { + cellId: 'cell-a', + cellUrl: cellAUrl, + enabled: true, + assignments: 1, + activityLeases: 1, + activityRequestUnits: 1, + restartBlockingActivityLeases: 1, + restartBlockingActivityRequestUnits: 1, + restartBlockingReservedRequests: 1, + runtime: { cellUrl: cellAUrl, ready: true, heartbeatFresh: true } + } + }) + const cellState = async (cellId: string, enabled: boolean): Promise => + await fetch(`${directorUrl}/v1/admin/cell-state`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId, enabled }) + }) + const configuredTarget = await fetch(`${directorUrl}/v1/admin/cell-config`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + cellId: 'cell-b', + cellUrl: cellBUrl, + capacityRequests: 10, + state: 'existing-only' + }) + }) + expect(configuredTarget.status).toBe(200) + expect(await configuredTarget.json()).toEqual({ ok: true }) + const incompleteLimit = await fetch(`${directorUrl}/v1/admin/cell-config`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + cellId: 'cell-b', + cellUrl: cellBUrl, + capacityRequests: 10, + connectionHardCap: 600, + enabled: false + }) + }) + expect(incompleteLimit.status).toBe(400) + const capacity = await fetch(`${directorUrl}/v1/admin/evacuation-capacity`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, sourceCellId: 'cell-a', targetCellId: 'cell-b' }) + }) + expect(capacity.status).toBe(200) + expect(await capacity.json()).toEqual({ + v: 1, + sourceAssignments: 1, + requiredTargetUnits: 2, + availableTargetUnits: 10 + }) + const disabledTarget = await fetch(`${directorUrl}/v1/admin/evacuate`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + userId: 'user-1', + relayHostId: hostId, + targetCellId: 'cell-b' + }) + }) + expect(disabledTarget.status).toBe(409) + expect(await disabledTarget.json()).toEqual({ error: 'target_cell_unavailable' }) + expect((await cellState('cell-b', true)).status).toBe(200) + const migrationResponse = await fetch(`${directorUrl}/v1/admin/evacuate-cell`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + limit: 10 + }) + }) + expect(migrationResponse.status).toBe(200) + expect(await migrationResponse.json()).toEqual({ v: 1, started: 1 }) + const pendingStatus = await fetch(`${directorUrl}/v1/admin/evacuation-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + sourceCellId: 'cell-a', + targetCellId: 'cell-b' + }) + }) + expect(pendingStatus.status).toBe(200) + expect(await pendingStatus.json()).toMatchObject({ + v: 1, + inProgress: 1, + targetRegistered: 0, + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 0, + blocked: 0, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 + }) + // Completion must prove the source has stopped admitting new work, even in + // the served admin workflow that performs its own topology preflight. + expect((await cellState('cell-a', false)).status).toBe(200) + + cellB = spawnTopologyRelay({ + url: cellBUrl, + dataDirectory, + role: 'cell', + cellId: 'cell-b' + }) + await waitForRelay(cellB) + relayUrl = cellBUrl + const targetHost = await openHostControl({ keyPair, assignmentEpoch: 2 }) + const completionBody = JSON.stringify({ + v: 1, + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + completeReady: true + }) + const premature = await fetch(`${directorUrl}/v1/admin/evacuation-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: completionBody + }) + expect(premature.status).toBe(200) + expect(await premature.json()).toMatchObject({ + v: 1, + inProgress: 1, + targetRegistered: 1, + registeredSourceActive: 1, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 0, + blocked: 1, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 + }) + + const sourceClosed = new Promise((resolveClose) => + sourceHost.socket.once('close', (code) => resolveClose(code)) + ) + const drain = await fetch(`${cellAUrl}/v1/admin/drain`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, graceMs: 0 }) + }) + expect(drain.status).toBe(200) + expect(await sourceClosed).toBe(4503) + + let completed: Response | undefined + for (let attempt = 0; attempt < 20; attempt++) { + completed = await fetch(`${directorUrl}/v1/admin/evacuation-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: completionBody + }) + if ( + completed.status === 200 && + ((await completed.clone().json()) as { inProgress?: number }).inProgress === 0 + ) { + break + } + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + expect(completed?.status).toBe(200) + expect(await completed?.json()).toMatchObject({ + v: 1, + inProgress: 0, + targetRegistered: 0, + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 1, + blocked: 0, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 + }) + const observedDatabase = await openRelayDatabase({ dataDir: dataDirectory }) + const assignment = await new RelayAssignmentStore(observedDatabase).resolve({ + userId: 'user-1', + relayHostId: hostId + }) + const reservations = await observedDatabase.query( + `SELECT cell_id, reserved_requests FROM relay_cells ORDER BY cell_id` + ) + await observedDatabase.close() + expect(assignment).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 }) + expect(reservations).toEqual([ + { cell_id: 'cell-a', reserved_requests: 0 }, + { cell_id: 'cell-b', reserved_requests: 1 } + ]) + const selectorStatusBefore = await fetch( + `${directorUrl}/v1/admin/admission-selector/status`, + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1 }) + } + ) + expect(selectorStatusBefore.status).toBe(200) + const selectorBefore = (await selectorStatusBefore.json()) as { + selector: { membership: CellAdmissionMembership } + } + const selectorInput = { + v: 1, + attemptId: 'blackbox_cutover', + expectedGeneration: 0, + expectedMembershipSha256: createHash('sha256') + .update(encodeMembership(selectorBefore.selector.membership)) + .digest('hex'), + membership: { + existingOnly: ['cell-a'], + migrationOnly: [], + general: ['cell-b'] + } + } + const unsafeSelectorInput = { ...selectorInput, expectedMembershipSha256: undefined } + const unsafeSelectorResponse = await fetch( + `${directorUrl}/v1/admin/admission-selector/apply`, + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(unsafeSelectorInput) + } + ) + expect(unsafeSelectorResponse.status).toBe(400) + const selectorResponse = await fetch( + `${directorUrl}/v1/admin/admission-selector/apply`, + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(selectorInput) + } + ) + expect(selectorResponse.status).toBe(200) + expect(await selectorResponse.json()).toMatchObject({ + v: 1, + changed: true, + selector: { generation: 1, membership: selectorInput.membership } + }) + const selectorStatus = await fetch( + `${directorUrl}/v1/admin/admission-selector/status`, + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, attemptId: selectorInput.attemptId }) + } + ) + expect(selectorStatus.status).toBe(200) + expect(await selectorStatus.json()).toMatchObject({ + v: 1, + selector: { generation: 1, membership: selectorInput.membership }, + intent: { attemptId: selectorInput.attemptId, state: 'committed' } + }) + const addCellsInput = { + v: 1, + attemptId: 'blackbox_add_cells', + expectedGeneration: 1, + cells: [ + { + cellId: 'cell-c', + cellUrl: 'https://relay-c.example.com', + capacityRequests: 20, + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + } + ] + } + const addCellsResponse = await fetch( + `${directorUrl}/v1/admin/admission-selector/add-migration-cells`, + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(addCellsInput) + } + ) + expect(addCellsResponse.status).toBe(200) + expect(await addCellsResponse.json()).toMatchObject({ + v: 1, + changed: true, + selector: { + generation: 2, + membership: { + existingOnly: ['cell-a'], + migrationOnly: ['cell-c'], + general: ['cell-b'] + } + } + }) + expect((await cellState('cell-a', true)).status).toBe(409) + targetHost.socket.close() + } finally { + for (const child of [cellA, cellB, director]) { + if (child && child.exitCode === null) child.kill('SIGKILL') + } + relayUrl = originalUrl + rmSync(dataDirectory, { recursive: true, force: true }) + } + }) + + it('recovers an unexpired invite through a restarted configured director without reservation', async () => { + const combinedUrl = relayUrl + const host = await openHostControl() + const hostId = createHash('sha256') + .update(host.keyPair.publicKey) + .digest('base64url') + .slice(0, 16) + const invitePromise = nextMessage(host.socket) + host.socket.send( + JSON.stringify({ type: 'invite-create', reqId: 'move-invite', relayDeviceId: 'move-device' }) + ) + const invite = await invitePromise + host.socket.close() + + relayProcess.kill('SIGKILL') + await new Promise((resolveExit) => relayProcess.once('exit', () => resolveExit())) + const directorPort = await unusedPort() + relayUrl = `http://127.0.0.1:${directorPort}` + relayProcess = spawn(process.execPath, ['--import', 'tsx', 'src/index.ts'], { + cwd: appDirectory, + env: { + ...process.env, + PORT: String(directorPort), + ORCA_RELAY_PUBLIC_URL: relayUrl, + ORCA_RELAY_CELL_URL: 'https://relay-c2.onorca.dev', + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: relayDataDirectory, + ORCA_RELAY_ROLE: 'director', + ORCA_RELAY_CELLS_JSON: JSON.stringify([ + { + id: 'cell-c2', + url: 'https://relay-c2.onorca.dev', + capacityRequests: 900 + } + ]), + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks` + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + await waitForRelay(relayProcess) + expect( + await postCellHeartbeat(relayUrl, { + id: 'cell-c2', + url: 'https://relay-c2.onorca.dev' + }) + ).toMatchObject({ status: 200 }) + + const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, { + headers: forwardedHeaders() + }) + await new Promise((resolveOpen) => phone.once('open', resolveOpen)) + const movedPromise = nextMessage(phone) + const closed = new Promise((resolveClose) => + phone.once('close', (code) => resolveClose(code)) + ) + phone.send( + JSON.stringify({ + type: 'relay-auth', + v: 1, + mode: 'connect', + credential: invite.inviteToken + }) + ) + expect(await movedPromise).toEqual({ + type: 'relay-moved', + v: 1, + cellUrl: 'https://relay-c2.onorca.dev', + assignmentEpoch: 1 + }) + expect(await closed).toBe(4503) + + relayProcess.kill('SIGTERM') + await new Promise((resolveExit) => relayProcess.once('exit', () => resolveExit())) + relayUrl = combinedUrl + relayProcess = spawn(process.execPath, ['--import', 'tsx', 'src/index.ts'], { + cwd: appDirectory, + env: { + ...process.env, + PORT: new URL(combinedUrl).port, + ORCA_RELAY_PUBLIC_URL: combinedUrl, + ORCA_RELAY_CELL_URL: combinedUrl, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: relayDataDirectory, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT: 'capacity@example.com', + ORCA_RELAY_MONITOR_SERVICE_ACCOUNT: 'monitor@example.com', + ORCA_RELAY_FENCE_SERVICE_ACCOUNT: 'fence@example.com', + ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`, + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks` + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + await waitForRelay(relayProcess) + }) + + it('uses configured-director recovery and typed 4503 on unplanned SIGTERM', async () => { + const originalUrl = relayUrl + const signalPort = await unusedPort() + const signalUrl = `http://127.0.0.1:${signalPort}` + const signalData = mkdtempSync(resolve(tmpdir(), 'orca-relay-sigterm-')) + const signalProcess = spawn(process.execPath, ['--import', 'tsx', 'src/index.ts'], { + cwd: appDirectory, + env: { + ...process.env, + PORT: String(signalPort), + ORCA_RELAY_PUBLIC_URL: signalUrl, + ORCA_RELAY_CELL_URL: signalUrl, + ORCA_RELAY_AUTH_ISSUER: issuer, + ORCA_RELAY_JWKS_URL: `${issuer}/jwks`, + ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: assignmentKey, + ORCA_RELAY_DATA_DIR: signalData, + ORCA_RELAY_ADMIN_AUDIENCE: adminAudience, + ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.com', + ORCA_RELAY_ADMIN_JWKS_URL: `${issuer}/jwks` + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + relayUrl = signalUrl + try { + await waitForRelay(signalProcess) + const host = await openHostControl() + const drain = nextMessage(host.socket) + const closed = new Promise((resolveClose) => + host.socket.once('close', (code) => resolveClose(code)) + ) + const exited = new Promise((resolveExit) => + signalProcess.once('exit', () => resolveExit()) + ) + signalProcess.kill('SIGTERM') + expect(await drain).toEqual({ + type: 'drain', + graceMs: 0, + recovery: 'resolve-director' + }) + expect(await closed).toBe(4503) + await exited + } finally { + if (signalProcess.exitCode === null) signalProcess.kill('SIGKILL') + relayUrl = originalUrl + rmSync(signalData, { recursive: true, force: true }) + } + }) + + it('authenticates admin drain with exact Google audience and deploy identity', async () => { + const host = await openHostControl() + const drainMessage = nextMessage(host.socket) + const closed = new Promise((resolveClose) => + host.socket.once('close', (code) => resolveClose(code)) + ) + const token = await new SignJWT({ email: 'deploy@example.com', email_verified: true }) + .setProtectedHeader({ alg: 'RS256', kid: 'admin-key' }) + .setIssuer('https://accounts.google.com') + .setAudience(adminAudience) + .setSubject('deploy-subject') + .setIssuedAt() + .setExpirationTime('5m') + .sign(adminPrivateKey) + const response = await fetch(`${relayUrl}/v1/admin/drain`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, graceMs: 0 }) + }) + expect(response.status).toBe(200) + expect(await drainMessage).toEqual({ + type: 'drain', + graceMs: 0, + recovery: 'resolve-director' + }) + expect(await closed).toBe(4503) + }) + + it('keeps dedicated capacity, monitor, and fence identities on exact routes', async () => { + const capacity = await googleServiceToken(adminAudience, 'capacity@example.com') + const monitor = await googleServiceToken(adminAudience, 'monitor@example.com') + const fence = await googleServiceToken(adminAudience, 'fence@example.com') + const broker = await googleServiceToken(adminAudience, 'broker@example.com') + const post = async (path: string, token: string, body: unknown): Promise => + await fetch(`${relayUrl}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(body) + }) + + expect((await post('/v1/admin/runtime-status', capacity, { v: 1 })).status).toBe(200) + expect( + (await post('/v1/admin/evacuation-status', capacity, { + v: 1, + sourceCellId: 'source', + targetCellId: 'target', + completeReady: false + })).status + ).toBe(401) + expect( + (await post('/v1/admin/cell-fence-attempt-status', capacity, { + v: 1, + cellId: 'production-gce-c1' + })).status + ).toBe(401) + + expect((await post('/v1/admin/runtime-status', monitor, { v: 1 })).status).toBe(200) + expect((await post('/v1/admin/drain', monitor, { v: 1, graceMs: 0 })).status).toBe(401) + expect( + (await post('/v1/admin/cell-fence-attempt-status', monitor, { + v: 1, + cellId: 'production-gce-c1' + })).status + ).toBe(401) + + expect((await post('/v1/admin/runtime-status', fence, { v: 1 })).status).toBe(200) + expect((await post('/v1/admin/drain', fence, { v: 1, graceMs: 0 })).status).toBe(401) + expect( + (await post('/v1/admin/cell-fence-attempt-status', fence, { + v: 1, + cellId: 'production-gce-c1' + })).status + ).toBe(404) + expect( + (await post('/v1/admin/cell-fence-attest', fence, { + v: 1 + })).status + ).toBe(401) + + const directorPort = await unusedPort() + const directorUrl = `http://127.0.0.1:${directorPort}` + const directorData = mkdtempSync(resolve(tmpdir(), 'orca-relay-monitor-auth-')) + const director = spawnTopologyRelay({ + url: directorUrl, + dataDirectory: directorData, + role: 'director', + cellId: 'director', + cells: [{ id: 'cell-a', url: 'https://cell-a.example.com', capacityRequests: 10 }] + }) + try { + await waitForRelay(director) + const adoption = { + v: 1, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + confirmation: 'ADOPT_LEGACY_TERRAFORM_CELL_FENCE' + } + const postDirector = async (token: string): Promise => + await fetch(`${directorUrl}/v1/admin/cell-fence-adopt-legacy`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(adoption) + }) + expect( + (await postDirector(fence)).status + ).toBe(401) + expect( + (await postDirector(broker)).status + ).toBe(409) + const commitAdoption = { + v: 1, + cellId: 'cell-a', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + confirmation: 'COMMIT_LEGACY_TERRAFORM_CELL_FENCE_ADOPTION' + } + const postCommit = async (token: string): Promise => + await fetch(`${directorUrl}/v1/admin/cell-fence-commit-legacy-adoption`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(commitAdoption) + }) + expect((await postCommit(fence)).status).toBe(401) + expect((await postCommit(broker)).status).toBe(409) + const response = await fetch(`${directorUrl}/v1/admin/evacuation-status`, { + method: 'POST', + headers: { + authorization: `Bearer ${monitor}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + v: 1, + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + completeReady: true + }) + }) + expect(response.status).toBe(403) + const fenceResponse = await fetch(`${directorUrl}/v1/admin/evacuation-status`, { + method: 'POST', + headers: { + authorization: `Bearer ${fence}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + v: 1, + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + completeReady: true + }) + }) + expect(fenceResponse.status).toBe(403) + } finally { + if (director.exitCode === null) director.kill('SIGKILL') + rmSync(directorData, { recursive: true, force: true }) + } + }) + + it('reports only authenticated aggregate runtime identity and the served digest', async () => { + const token = await adminToken() + expect( + await fetch(`${relayUrl}/v1/admin/runtime-status`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1 }) + }) + ).toMatchObject({ status: 401 }) + expect( + await fetch(`${relayUrl}/v1/admin/runtime-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, hostId: 'must-not-be-accepted' }) + }) + ).toMatchObject({ status: 400 }) + const response = await fetch(`${relayUrl}/v1/admin/runtime-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1 }) + }) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + v: 1, + role: 'combined', + cellId: 'combined', + cellUrl: relayUrl, + region: 'us-central1', + imageDigest: `sha256:${'a'.repeat(64)}`, + draining: true, + regionalRehomeProtocol: 0, + connectionCapacity: null, + runtime: { + totalConnections: 0, + preAuthConnections: 0, + controls: 0, + splices: 0, + pendingSplices: 0, + queuedBytes: 0 + } + }) + }) +}) diff --git a/cloud/apps/relay/src/splice-forwarder.test.ts b/cloud/apps/relay/src/splice-forwarder.test.ts new file mode 100644 index 00000000000..3aa5e75fc97 --- /dev/null +++ b/cloud/apps/relay/src/splice-forwarder.test.ts @@ -0,0 +1,135 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' +import type WebSocket from 'ws' +import { + ProcessQueuedByteBudget, + wireSplice, + type SpliceCloseInfo +} from './splice-forwarder.js' + +class FakeSocket extends EventEmitter { + readonly OPEN = 1 + readyState = 1 + bufferedAmount = 0 + sent: Array<{ data: unknown; binary: boolean }> = [] + closes: Array<{ code: number; reason: string }> = [] + _socket = { pause: vi.fn(), resume: vi.fn(), setNoDelay: vi.fn() } + + send(data: unknown, options: { binary: boolean }): void { + this.sent.push({ data, binary: options.binary }) + } + + close(code: number, reason: string): void { + this.readyState = 3 + this.closes.push({ code, reason }) + this.emit('close', code, Buffer.from(reason)) + } +} + +describe('splice forwarder', () => { + it('preserves text/binary opcodes and propagates peer close', () => { + const client = new FakeSocket() + const host = new FakeSocket() + const onClose = vi.fn() + const onForwardedBytes = vi.fn() + wireSplice({ + client: client as unknown as WebSocket, + host: host as unknown as WebSocket, + budget: new ProcessQueuedByteBudget(), + onClose, + onForwardedBytes + }) + client.emit('message', Buffer.from('text'), false) + client.emit('message', Buffer.from([1, 2]), true) + expect(host.sent).toEqual([ + { data: Buffer.from('text'), binary: false }, + { data: Buffer.from([1, 2]), binary: true } + ]) + expect(onForwardedBytes.mock.calls).toEqual([[4], [2]]) + client.emit('close', 1000, Buffer.alloc(0)) + expect(host.closes[0]?.code).toBe(4408) + expect(onClose).toHaveBeenCalledOnce() + }) + + it('hard-closes a wedged splice and releases its global queued-byte reservation', () => { + const client = new FakeSocket() + const host = new FakeSocket() + host.bufferedAmount = 1024 * 1024 + const budget = new ProcessQueuedByteBudget() + wireSplice({ + client: client as unknown as WebSocket, + host: host as unknown as WebSocket, + budget, + onClose: vi.fn() + }) + client.emit('message', Buffer.alloc(8 * 1024 * 1024), true) + expect(budget.current()).toBe(8 * 1024 * 1024) + client.emit('message', Buffer.alloc(512 * 1024), true) + expect(client.closes[0]?.code).toBe(4429) + expect(host.closes[0]?.code).toBe(4429) + expect(budget.current()).toBe(0) + }) + + it('reports the close trigger so limit and oversize kills are attributable', () => { + const wire = (): { client: FakeSocket; host: FakeSocket; closes: SpliceCloseInfo[] } => { + const client = new FakeSocket() + const host = new FakeSocket() + const closes: SpliceCloseInfo[] = [] + wireSplice({ + client: client as unknown as WebSocket, + host: host as unknown as WebSocket, + budget: new ProcessQueuedByteBudget(), + onClose: vi.fn(), + onClosed: (closeInfo) => closes.push(closeInfo) + }) + return { client, host, closes } + } + + const queueLimit = wire() + queueLimit.host.bufferedAmount = 1024 * 1024 + queueLimit.client.emit('message', Buffer.alloc(8 * 1024 * 1024), true) + queueLimit.client.emit('message', Buffer.alloc(512 * 1024), true) + expect(queueLimit.closes).toEqual([ + { code: 4429, reason: 'relay queue limit exceeded', trigger: 'queue-limit' } + ]) + + // The ws receiver error for a frame above maxPayload names the payload cap. + const oversize = wire() + oversize.host.emit('error', new RangeError('Max payload size exceeded')) + expect(oversize.closes[0]?.trigger).toBe('host-oversize-frame') + + const peerClose = wire() + peerClose.client.emit('close', 1001, Buffer.alloc(0)) + expect(peerClose.closes[0]?.trigger).toBe('client-closed') + expect(peerClose.closes).toHaveLength(1) + }) + + it('queues one full catalog-sized frame for a backpressured peer without closing', async () => { + // Why: the desktop's worktree catalog response exceeds 1MiB on large + // workspaces; a single maxFrameBytes frame must survive backpressure. + const client = new FakeSocket() + const host = new FakeSocket() + host.bufferedAmount = 1024 * 1024 + const budget = new ProcessQueuedByteBudget() + const onClose = vi.fn() + wireSplice({ + client: client as unknown as WebSocket, + host: host as unknown as WebSocket, + budget, + onClose + }) + client.emit('message', Buffer.alloc(8 * 1024 * 1024), true) + expect(client.closes).toEqual([]) + expect(host.closes).toEqual([]) + expect(budget.current()).toBe(8 * 1024 * 1024) + + // Peer drains; the queued frame flushes and the reservation releases. + host.bufferedAmount = 0 + await new Promise((resolve) => setTimeout(resolve, 60)) + expect(host.sent.some((frame) => (frame.data as Buffer).byteLength === 8 * 1024 * 1024)).toBe( + true + ) + expect(budget.current()).toBe(0) + expect(onClose).not.toHaveBeenCalled() + }) +}) diff --git a/cloud/apps/relay/src/splice-forwarder.ts b/cloud/apps/relay/src/splice-forwarder.ts new file mode 100644 index 00000000000..b0a31b49830 --- /dev/null +++ b/cloud/apps/relay/src/splice-forwarder.ts @@ -0,0 +1,158 @@ +import { RELAY_ADMISSION_BUDGETS, RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract' +import type WebSocket from 'ws' +import type { RawData } from 'ws' +import { closeRelayWebSocket } from './relay-websocket-close.js' + +type QueuedFrame = { data: RawData; binary: boolean; bytes: number } + +export class ProcessQueuedByteBudget { + private queued = 0 + + reserve(bytes: number): boolean { + if (this.queued + bytes > RELAY_ADMISSION_BUDGETS.maxProcessQueuedBytes) return false + this.queued += bytes + return true + } + + release(bytes: number): void { + this.queued = Math.max(0, this.queued - bytes) + } + + current(): number { + return this.queued + } +} + +function frameBytes(data: RawData): number { + if (typeof data === 'string') return Buffer.byteLength(data) + if (Array.isArray(data)) return data.reduce((total, part) => total + part.byteLength, 0) + return data.byteLength +} + +function transport(socket: WebSocket): { pause(): void; resume(): void; setNoDelay(value: boolean): void } | null { + return ( + socket as WebSocket & { + _socket?: { pause(): void; resume(): void; setNoDelay(value: boolean): void } + } + )._socket ?? null +} + +export type SpliceCloseInfo = { code: number; reason: string; trigger: string } + +// The ws receiver kills a connection whose frame exceeds maxPayload with this +// message; surfacing it separately is what makes catalog-growth kills visible. +function errorTrigger(side: 'client' | 'host', error: Error): string { + return /max payload/i.test(error.message) ? `${side}-oversize-frame` : `${side}-error` +} + +export function wireSplice(input: { + client: WebSocket + host: WebSocket + budget: ProcessQueuedByteBudget + onClose: () => void + onForwardedBytes?: (bytes: number) => void + onClosed?: (closeInfo: SpliceCloseInfo) => void +}): (code?: number, reason?: string) => void { + let closed = false + const timers = new Set>() + const cleanups: Array<() => void> = [] + + const close = ( + code: number = RELAY_CLOSE_CODE.PEER_DROPPED, + reason = 'peer connection dropped', + trigger = 'external' + ): void => { + if (closed) return + closed = true + for (const timer of timers) clearTimeout(timer) + timers.clear() + for (const cleanup of cleanups) cleanup() + closeRelayWebSocket(input.client, code, reason) + closeRelayWebSocket(input.host, code, reason) + input.onClosed?.({ code, reason, trigger }) + input.onClose() + } + + const direction = (source: WebSocket, target: WebSocket): void => { + const queue: QueuedFrame[] = [] + let queuedBytes = 0 + let wedgedSince: number | null = null + cleanups.push(() => { + input.budget.release(queuedBytes) + queuedBytes = 0 + queue.length = 0 + transport(source)?.resume() + }) + + const flush = (): void => { + if (closed) return + if (target.readyState !== target.OPEN || source.readyState !== source.OPEN) { + close(undefined, undefined, 'peer-gone') + return + } + while ( + queue.length > 0 && + target.bufferedAmount <= RELAY_ADMISSION_BUDGETS.spliceLowWaterBytes + ) { + const frame = queue.shift()! + queuedBytes -= frame.bytes + input.budget.release(frame.bytes) + target.send(frame.data, { binary: frame.binary }) + } + if (queue.length === 0) { + wedgedSince = null + transport(source)?.resume() + return + } + if ( + wedgedSince !== null && + Date.now() - wedgedSince >= RELAY_ADMISSION_BUDGETS.spliceWedgedTimeoutMs + ) { + close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'wedged relay link', 'wedged') + return + } + const timer = setTimeout(() => { + timers.delete(timer) + flush() + }, 25) + timers.add(timer) + } + + source.on('message', (data, binary) => { + if (closed) return + const bytes = frameBytes(data) + if ( + queue.length === 0 && + target.readyState === target.OPEN && + target.bufferedAmount <= RELAY_ADMISSION_BUDGETS.spliceHighWaterBytes + ) { + target.send(data, { binary }) + input.onForwardedBytes?.(bytes) + return + } + if ( + queuedBytes + bytes > RELAY_ADMISSION_BUDGETS.spliceHardQueuedBytes || + !input.budget.reserve(bytes) + ) { + close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'relay queue limit exceeded', 'queue-limit') + return + } + queue.push({ data, binary, bytes }) + queuedBytes += bytes + input.onForwardedBytes?.(bytes) + wedgedSince ??= Date.now() + transport(source)?.pause() + if (queue.length === 1) flush() + }) + } + + transport(input.client)?.setNoDelay(true) + transport(input.host)?.setNoDelay(true) + direction(input.client, input.host) + direction(input.host, input.client) + input.client.once('close', () => close(undefined, undefined, 'client-closed')) + input.host.once('close', () => close(undefined, undefined, 'host-closed')) + input.client.once('error', (error) => close(undefined, undefined, errorTrigger('client', error))) + input.host.once('error', (error) => close(undefined, undefined, errorTrigger('host', error))) + return close +} diff --git a/cloud/apps/relay/src/staging-asia-proof-admission.test.ts b/cloud/apps/relay/src/staging-asia-proof-admission.test.ts new file mode 100644 index 00000000000..3bbee3ab9a3 --- /dev/null +++ b/cloud/apps/relay/src/staging-asia-proof-admission.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest' +import { stagingAsiaProofMembership } from './staging-asia-proof-admission.js' + +describe('staging Asia proof admission', () => { + it('promotes only C4 and preserves every other cell', () => { + expect(stagingAsiaProofMembership({ + existingOnly: ['staging-gce-c1'], + migrationOnly: ['staging-gce-c4'], + general: ['staging-gce-c2'] + }, 'general')).toEqual({ + existingOnly: ['staging-gce-c1'], + migrationOnly: [], + general: ['staging-gce-c2', 'staging-gce-c4'] + }) + }) + + it('rolls back only C4', () => { + expect(stagingAsiaProofMembership({ + existingOnly: ['staging-gce-c1'], + migrationOnly: [], + general: ['staging-gce-c2', 'staging-gce-c4'] + }, 'migration-only')).toEqual({ + existingOnly: ['staging-gce-c1'], + migrationOnly: ['staging-gce-c4'], + general: ['staging-gce-c2'] + }) + }) + + it('rejects absent or irreversible C4 state', () => { + expect(() => stagingAsiaProofMembership({ + existingOnly: [], migrationOnly: [], general: ['staging-gce-c2'] + }, 'general')).toThrow('staging_asia_proof_cell_not_transitionable') + expect(() => stagingAsiaProofMembership({ + existingOnly: ['staging-gce-c4'], migrationOnly: [], general: [] + }, 'migration-only')).toThrow('staging_asia_proof_cell_not_transitionable') + }) +}) diff --git a/cloud/apps/relay/src/staging-asia-proof-admission.ts b/cloud/apps/relay/src/staging-asia-proof-admission.ts new file mode 100644 index 00000000000..a45a338768d --- /dev/null +++ b/cloud/apps/relay/src/staging-asia-proof-admission.ts @@ -0,0 +1,30 @@ +import type { CellAdmissionMembership } from './cell-admission-selector.js' + +export const STAGING_ASIA_PROOF_CELL_ID = 'staging-gce-c4' + +export type StagingAsiaProofAdmissionState = 'general' | 'migration-only' + +export function stagingAsiaProofMembership( + current: CellAdmissionMembership, + state: StagingAsiaProofAdmissionState +): CellAdmissionMembership { + const currentStates = [ + current.existingOnly.includes(STAGING_ASIA_PROOF_CELL_ID), + current.migrationOnly.includes(STAGING_ASIA_PROOF_CELL_ID), + current.general.includes(STAGING_ASIA_PROOF_CELL_ID) + ] + if (currentStates.filter(Boolean).length !== 1 || currentStates[0]) { + throw new Error('staging_asia_proof_cell_not_transitionable') + } + return { + existingOnly: [...current.existingOnly], + migrationOnly: current.migrationOnly + .filter((cellId) => cellId !== STAGING_ASIA_PROOF_CELL_ID) + .concat(state === 'migration-only' ? [STAGING_ASIA_PROOF_CELL_ID] : []) + .sort(), + general: current.general + .filter((cellId) => cellId !== STAGING_ASIA_PROOF_CELL_ID) + .concat(state === 'general' ? [STAGING_ASIA_PROOF_CELL_ID] : []) + .sort() + } +} diff --git a/cloud/apps/relay/tsconfig.build.json b/cloud/apps/relay/tsconfig.build.json new file mode 100644 index 00000000000..489ddfd34d6 --- /dev/null +++ b/cloud/apps/relay/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "noEmit": false, + "outDir": "dist", + "rootDir": "src" + }, + "exclude": ["src/**/*.test.ts"] +} diff --git a/cloud/apps/relay/tsconfig.json b/cloud/apps/relay/tsconfig.json new file mode 100644 index 00000000000..a552e34dbe9 --- /dev/null +++ b/cloud/apps/relay/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true }, + "include": ["src/**/*.ts"] +} diff --git a/cloud/apps/relay/vitest.config.ts b/cloud/apps/relay/vitest.config.ts new file mode 100644 index 00000000000..f56bbd7be3a --- /dev/null +++ b/cloud/apps/relay/vitest.config.ts @@ -0,0 +1,44 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { defaultExclude, defineConfig } from 'vitest/config' + +// Every test file that opens ORCA_RELAY_TEST_POSTGRES_URL shares one CI +// database, and those tests take session-level locks with a 1s lock_timeout. +// Running them alongside anything else collides into lock timeouts, capacity +// exhaustion, and afterAll hangs. Keep them in their own serialized project so +// only they give up file parallelism; the rest of the suite opens SQLite data +// directories and stays fully parallel. +const sourceDirectory = fileURLToPath(new URL('src', import.meta.url)) +const sharedPostgresTests = readdirSync(sourceDirectory) + .filter((entry) => entry.endsWith('.test.ts')) + .filter((entry) => + readFileSync(`${sourceDirectory}/${entry}`, 'utf8').includes( + 'ORCA_RELAY_TEST_POSTGRES_URL' + ) + ) + .map((entry) => `src/${entry}`) + +const timeouts = { testTimeout: 15_000, hookTimeout: 15_000 } + +export default defineConfig({ + test: { + projects: [ + { + test: { + ...timeouts, + name: 'relay', + include: ['src/**/*.test.ts'], + exclude: [...defaultExclude, ...sharedPostgresTests] + } + }, + { + test: { + ...timeouts, + name: 'relay-postgres', + include: sharedPostgresTests, + fileParallelism: false + } + } + ] + } +}) diff --git a/cloud/dev/contracts/production-cloud-sql-app-consumers.json b/cloud/dev/contracts/production-cloud-sql-app-consumers.json new file mode 100644 index 00000000000..1cf4ba600dc --- /dev/null +++ b/cloud/dev/contracts/production-cloud-sql-app-consumers.json @@ -0,0 +1,15 @@ +{ + "comment": "Production Cloud SQL consumers owned by the private orca-cloud application tree (auth and API services). The relay ships without them, so the values the connection budget needs are published here; the private repository binds every field back to its source in its own CI.", + "authInstances": 2, + "authPoolMax": 10, + "apiInstances": 10, + "apiPoolMax": 5, + "maxConnections": 400, + "sources": { + "authInstances": "private apps tfvars: auth service max instances", + "authPoolMax": "private auth service: pg.Pool max", + "apiInstances": "private apps tfvars: API service max instances", + "apiPoolMax": "private API service: pg.Pool max", + "maxConnections": "Cloud SQL tier default; no max_connections flag is set" + } +} diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json new file mode 100644 index 00000000000..9664c1eb299 --- /dev/null +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -0,0 +1,267 @@ +{ + "comment": "Terraform root ownership for every resource family declared under infra/terraform (relay), infra/terraform-foundation, and infra/terraform-apps. env_conditional families live in relay for production and in apps for staging (complementary counts). never_in_relay_state lists foundation/apps families that were introduced with their new root and so must NOT appear in infra/terraform/relay-root-carve-removed.tf. Generated by dev/scripts/terraform-root-partition.mjs --write; the test asserts this file matches the declarations.", + "foundation": [ + "google_artifact_registry_repository.api", + "google_iam_workload_identity_pool.github", + "google_project_service.required", + "google_project_service.sqladmin", + "google_service_account.runtime", + "google_sql_database_instance.auth", + "google_storage_bucket_iam_member.cloud_sql_rollout_lease", + "google_storage_bucket_iam_member.cloud_sql_rollout_lease_bucket_reader" + ], + "apps": [ + "cloudflare_record.artifact_onorca", + "cloudflare_record.artifact_usercontent", + "cloudflare_record.auth", + "google_artifact_registry_repository_iam_member.github_production_app_deploy_writer", + "google_cloud_run_domain_mapping.artifacts", + "google_cloud_run_domain_mapping.auth", + "google_cloud_run_v2_job.skill_storage_monitor", + "google_cloud_run_v2_job_iam_member.github_production_app_skill_monitor_developer", + "google_cloud_run_v2_job_iam_member.skill_storage_scheduler_invoker", + "google_cloud_run_v2_service.api", + "google_cloud_run_v2_service.auth", + "google_cloud_run_v2_service_iam_member.github_production_app_api_developer", + "google_cloud_run_v2_service_iam_member.github_production_app_auth_developer", + "google_cloud_scheduler_job.skill_storage_monitor", + "google_iam_workload_identity_pool_provider.github_production_app_deploy", + "google_logging_metric.skill_api", + "google_logging_metric.skill_archive_rejection", + "google_logging_metric.skill_database", + "google_logging_metric.skill_digest_mismatch", + "google_logging_metric.skill_finalize_saturation", + "google_logging_metric.skill_route_latency", + "google_logging_metric.skill_signing_failure", + "google_logging_metric.skill_storage_inventory", + "google_logging_metric.skill_storage_overdue_quarantine", + "google_logging_project_exclusion.skill_share_bearer_request_urls", + "google_monitoring_alert_policy.skill_api_5xx", + "google_monitoring_alert_policy.skill_cloud_run_resource_pressure", + "google_monitoring_alert_policy.skill_database_migration_failure", + "google_monitoring_alert_policy.skill_digest_mismatch", + "google_monitoring_alert_policy.skill_finalize_failures", + "google_monitoring_alert_policy.skill_route_latency", + "google_monitoring_alert_policy.skill_signing_failure", + "google_monitoring_alert_policy.skill_storage_lifecycle_failure", + "google_monitoring_dashboard.skill_sharing", + "google_project_iam_custom_role.skill_storage_inventory", + "google_project_iam_member.auth_runtime_cloudsql_client", + "google_project_iam_member.github_artifact_writer", + "google_project_iam_member.github_cloud_run_developer", + "google_project_iam_member.runtime_skills_cloudsql_client", + "google_secret_manager_secret.auth", + "google_secret_manager_secret.auth_database_url", + "google_secret_manager_secret.skills_database_url", + "google_secret_manager_secret_iam_member.auth_database_url_accessor", + "google_secret_manager_secret_iam_member.auth_runtime_accessor", + "google_secret_manager_secret_iam_member.runtime_skills_database_url_accessor", + "google_secret_manager_secret_version.auth_database_url", + "google_secret_manager_secret_version.skills_database_url", + "google_service_account.auth_runtime", + "google_service_account.github_production_app_deploy", + "google_service_account.skill_storage_monitor", + "google_service_account.skill_storage_scheduler", + "google_service_account_iam_member.github_auth_runtime_service_account_user", + "google_service_account_iam_member.github_production_app_auth_runtime_user", + "google_service_account_iam_member.github_production_app_deploy_workload_identity_user", + "google_service_account_iam_member.github_production_app_runtime_user", + "google_service_account_iam_member.github_production_app_skill_monitor_user", + "google_service_account_iam_member.github_runtime_service_account_user", + "google_service_account_iam_member.github_skill_storage_monitor_user", + "google_service_account_iam_member.runtime_skill_package_signer", + "google_sql_database.auth", + "google_sql_database.skills", + "google_sql_user.auth", + "google_sql_user.skills", + "google_storage_bucket.artifacts", + "google_storage_bucket.skill_packages", + "google_storage_bucket_iam_member.runtime_artifact_object_admin", + "google_storage_bucket_iam_member.runtime_skill_package_object_user", + "google_storage_bucket_iam_member.skill_storage_monitor_inventory", + "random_password.auth_database", + "random_password.skills_database", + "time_sleep.skill_metric_descriptor_propagation" + ], + "relay": [ + "google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer", + "google_artifact_registry_repository_iam_member.github_production_relay_writer", + "google_artifact_registry_repository_iam_member.github_relay_asia_topology_artifact_reader", + "google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader", + "google_certificate_manager_certificate.relay_gce", + "google_certificate_manager_certificate_map.relay_gce", + "google_certificate_manager_certificate_map_entry.relay_gce", + "google_certificate_manager_dns_authorization.relay_gce", + "google_cloud_run_domain_mapping.relay", + "google_cloud_run_domain_mapping.relay_cell", + "google_cloud_run_v2_service.relay", + "google_cloud_run_v2_service.relay_cell", + "google_cloud_run_v2_service.relay_fence_broker", + "google_cloud_run_v2_service_iam_member.github_production_relay_director_developer", + "google_cloud_run_v2_service_iam_member.github_production_relay_fence_broker_developer", + "google_cloud_run_v2_service_iam_member.github_staging_relay_capacity_developer", + "google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer", + "google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer", + "google_cloud_run_v2_service_iam_member.relay_fence_broker_deploy_invoker", + "google_cloud_run_v2_service_iam_member.relay_fence_broker_invoker", + "google_compute_backend_service.relay_gce_cell", + "google_compute_firewall.relay_gce_iap_ssh", + "google_compute_firewall.relay_gce_load_balancer", + "google_compute_global_address.relay_gce", + "google_compute_global_forwarding_rule.relay_gce", + "google_compute_health_check.relay_gce_liveness", + "google_compute_health_check.relay_gce_readiness", + "google_compute_instance_group_manager.relay_gce_cell", + "google_compute_instance_template.relay_gce_cell", + "google_compute_network.relay_gce", + "google_compute_router.relay_gce", + "google_compute_router.relay_gce_additional", + "google_compute_router_nat.relay_gce", + "google_compute_router_nat.relay_gce_additional", + "google_compute_subnetwork.relay_gce", + "google_compute_subnetwork.relay_gce_additional", + "google_compute_target_https_proxy.relay_gce", + "google_compute_url_map.relay_gce", + "google_iam_workload_identity_pool_provider.github_fence", + "google_iam_workload_identity_pool_provider.github_monitor", + "google_iam_workload_identity_pool_provider.github_production_relay_capacity", + "google_iam_workload_identity_pool_provider.github_relay_asia_proof", + "google_iam_workload_identity_pool_provider.github_relay_asia_topology", + "google_iam_workload_identity_pool_provider.github_staging_relay_capacity", + "google_iam_workload_identity_pool_provider.github_staging_relay_deploy", + "google_logging_metric.relay_incident", + "google_logging_metric.relay_snapshot", + "google_monitoring_alert_policy.relay_assignment_5xx", + "google_monitoring_alert_policy.relay_assignment_edge_429", + "google_monitoring_alert_policy.relay_cloud_sql_backends", + "google_monitoring_alert_policy.relay_custom", + "google_monitoring_alert_policy.relay_gce_connection_headroom", + "google_monitoring_alert_policy.relay_postgres_retry_exhausted", + "google_project_iam_custom_role.github_production_relay_capacity_mutation", + "google_project_iam_custom_role.github_relay_asia_topology_mutation", + "google_project_iam_custom_role.github_relay_asia_topology_read", + "google_project_iam_custom_role.github_relay_asia_topology_state_list", + "google_project_iam_custom_role.github_staging_relay_capacity_mutation", + "google_project_iam_custom_role.github_staging_relay_power", + "google_project_iam_custom_role.relay_fence_broker_mutation", + "google_project_iam_member.github_fence_cloudsql_viewer", + "google_project_iam_member.github_fence_compute_viewer", + "google_project_iam_member.github_fence_logging_viewer", + "google_project_iam_member.github_fence_monitoring_viewer", + "google_project_iam_member.github_monitor_compute_viewer", + "google_project_iam_member.github_production_relay_capacity_artifact_reader", + "google_project_iam_member.github_production_relay_capacity_mutation", + "google_project_iam_member.github_production_relay_capacity_viewer", + "google_project_iam_member.github_relay_asia_proof_logging_viewer", + "google_project_iam_member.github_relay_asia_proof_monitoring_viewer", + "google_project_iam_member.github_relay_asia_topology_mutation", + "google_project_iam_member.github_relay_asia_topology_read", + "google_project_iam_member.github_relay_monitor_cloudsql_viewer", + "google_project_iam_member.github_relay_monitor_logging_viewer", + "google_project_iam_member.github_relay_monitor_monitoring_viewer", + "google_project_iam_member.github_staging_relay_capacity_artifact_reader", + "google_project_iam_member.github_staging_relay_capacity_mutation", + "google_project_iam_member.github_staging_relay_capacity_viewer", + "google_project_iam_member.github_staging_relay_deploy_compute_viewer", + "google_project_iam_member.github_staging_relay_power", + "google_project_iam_member.relay_director_runtime_cloudsql_client", + "google_project_iam_member.relay_fence_broker_artifact_reader", + "google_project_iam_member.relay_fence_broker_compute_viewer", + "google_project_iam_member.relay_fence_broker_logging_viewer", + "google_project_iam_member.relay_fence_broker_mutation", + "google_project_iam_member.relay_runtime_artifact_reader", + "google_project_iam_member.relay_runtime_cloudsql_client", + "google_project_iam_member.relay_runtime_log_writer", + "google_secret_manager_secret.relay_assignment_signing_key", + "google_secret_manager_secret.relay_database_url", + "google_secret_manager_secret.relay_regional_placement_enabled", + "google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor", + "google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor", + "google_secret_manager_secret_iam_member.relay_database_url_accessor", + "google_secret_manager_secret_iam_member.relay_database_url_director_accessor", + "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor", + "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder", + "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer", + "google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor", + "google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor", + "google_secret_manager_secret_version.relay_assignment_signing_key", + "google_secret_manager_secret_version.relay_database_url", + "google_secret_manager_secret_version.relay_regional_placement_enabled", + "google_service_account.github_fence", + "google_service_account.github_monitor", + "google_service_account.github_production_relay_capacity", + "google_service_account.github_relay_asia_proof", + "google_service_account.github_relay_asia_topology", + "google_service_account.github_staging_relay_capacity", + "google_service_account.github_staging_relay_deploy", + "google_service_account.relay_director_runtime", + "google_service_account.relay_fence_broker", + "google_service_account.relay_runtime", + "google_service_account_iam_member.github_accepted_repository_workload_identity_user", + "google_service_account_iam_member.github_fence_workload_identity_user", + "google_service_account_iam_member.github_monitor_workload_identity_user", + "google_service_account_iam_member.github_production_relay_capacity_runtime_user", + "google_service_account_iam_member.github_production_relay_capacity_workload_identity_user", + "google_service_account_iam_member.github_relay_asia_proof_workload_identity_user", + "google_service_account_iam_member.github_relay_asia_topology_runtime_user", + "google_service_account_iam_member.github_relay_asia_topology_workload_identity_user", + "google_service_account_iam_member.github_relay_director_runtime_service_account_user", + "google_service_account_iam_member.github_relay_fence_broker_service_account_user", + "google_service_account_iam_member.github_relay_runtime_service_account_user", + "google_service_account_iam_member.github_staging_relay_capacity_runtime_user", + "google_service_account_iam_member.github_staging_relay_capacity_workload_identity_user", + "google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user", + "google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user", + "google_service_account_iam_member.relay_fence_broker_requester_token_creator", + "google_sql_database.relay", + "google_sql_user.relay", + "google_storage_bucket_iam_member.github_production_relay_capacity_state", + "google_storage_bucket_iam_member.github_relay_asia_topology_state", + "google_storage_bucket_iam_member.github_relay_asia_topology_state_list", + "google_storage_bucket_iam_member.github_staging_relay_capacity_state", + "google_storage_bucket_iam_member.github_staging_relay_deploy_state", + "google_storage_bucket_iam_member.github_staging_relay_deploy_state_list", + "google_storage_bucket_iam_member.relay_fence_broker_bucket_reader", + "google_storage_bucket_iam_member.relay_fence_broker_state_objects", + "random_password.relay_assignment_signing_key", + "random_password.relay_database" + ], + "env_conditional": { + "google_iam_workload_identity_pool_provider.github": { + "production": "relay", + "staging": "apps" + }, + "google_project_iam_member.github_cloudsql_viewer": { + "production": "relay", + "staging": "apps" + }, + "google_project_iam_member.github_compute_viewer": { + "production": "relay", + "staging": "apps" + }, + "google_project_iam_member.github_logging_viewer": { + "production": "relay", + "staging": "apps" + }, + "google_project_iam_member.github_monitoring_viewer": { + "production": "relay", + "staging": "apps" + }, + "google_service_account.github_deploy": { + "production": "relay", + "staging": "apps" + }, + "google_service_account_iam_member.github_workload_identity_user": { + "production": "relay", + "staging": "apps" + }, + "google_storage_bucket_iam_member.github_terraform_state_reader": { + "production": "relay", + "staging": "apps" + } + }, + "state_orphans": { + "staging": [], + "production": [] + } +} diff --git a/cloud/dev/scripts/capture-terraform-plan-baseline.mjs b/cloud/dev/scripts/capture-terraform-plan-baseline.mjs new file mode 100644 index 00000000000..734d67536c2 --- /dev/null +++ b/cloud/dev/scripts/capture-terraform-plan-baseline.mjs @@ -0,0 +1,83 @@ +#!/usr/bin/env node +import { execFileSync } from 'node:child_process' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +// Why: the relay root can never plan zero-diff (cell templates roll one at a time by design), +// so the split is gated on plan EQUIVALENCE: the normalized change set for a root's addresses +// must be identical before and after a state move. This captures that set deterministically. +// Read-only: -lock=false, -refresh=false, no apply. Forgets from `removed` blocks are excluded +// because the baseline has none. + +const usage = + 'usage: capture-terraform-plan-baseline.mjs --root --env --out [--tag ]' + +export function normalizePlan(planJson) { + const changes = (planJson.resource_changes ?? []) + .filter((entry) => !entry.change.actions.includes('forget')) + .map((entry) => ({ + address: entry.address, + actions: entry.change.actions, + before: entry.change.before ?? null, + after: entry.change.after ?? null, + after_unknown: entry.change.after_unknown ?? null + })) + .sort((left, right) => (left.address < right.address ? -1 : left.address > right.address ? 1 : 0)) + return changes +} + +export function summarize(changes) { + const counts = { create: 0, update: 0, delete: 0, replace: 0, 'no-op': 0, read: 0 } + for (const change of changes) { + const key = change.actions.join('-') + if (key === 'create') counts.create += 1 + else if (key === 'update') counts.update += 1 + else if (key === 'delete') counts.delete += 1 + else if (key === 'delete-create' || key === 'create-delete') counts.replace += 1 + else if (key === 'read') counts.read += 1 + else counts['no-op'] += 1 + } + return counts +} + +function argument(flag) { + const index = process.argv.indexOf(flag) + return index >= 0 ? process.argv[index + 1] : undefined +} + +if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split('/').pop())) { + const root = argument('--root') + const environment = argument('--env') + const out = argument('--out') + const tag = argument('--tag') ?? `${environment}-${root.replaceAll('/', '_')}` + if (!root || !['staging', 'production'].includes(environment) || !out) { + process.stderr.write(`${usage}\n`) + process.exit(2) + } + // The Cloudflare override only applies to the root that still declares the records; the relay + // root dropped them in the carve and errors on a -var for an undeclared variable. + const declaresArtifactDns = readFileSync(join(root, 'variables.tf'), 'utf8').includes( + 'variable "manage_artifact_dns"' + ) + mkdirSync(out, { recursive: true }) + const planFile = join(out, `${tag}.tfplan`) + execFileSync( + 'terraform', + [ + `-chdir=${root}`, 'plan', '-input=false', '-lock=false', '-refresh=false', '-no-color', + `-var-file=environments/${environment}.tfvars`, + ...(declaresArtifactDns ? ['-var', 'manage_artifact_dns=false'] : []), + `-out=${planFile}` + ], + { stdio: ['ignore', 'inherit', 'inherit'] } + ) + const json = JSON.parse( + execFileSync('terraform', [`-chdir=${root}`, 'show', '-json', planFile], { + encoding: 'utf8', + maxBuffer: 256 * 1024 * 1024 + }) + ) + const normalized = normalizePlan(json) + writeFileSync(join(out, `${tag}.norm.json`), `${JSON.stringify(normalized, null, 1)}\n`) + process.stdout.write(`${tag}: ${JSON.stringify(summarize(normalized))}\n`) +} diff --git a/cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs b/cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs new file mode 100644 index 00000000000..b003f3db773 --- /dev/null +++ b/cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs @@ -0,0 +1,15 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { normalizePlan, summarize } from './capture-terraform-plan-baseline.mjs' + +test('normalizes, sorts, and drops forgets so a removed block cannot skew equivalence', () => { + const changes = normalizePlan({ + resource_changes: [ + { address: 'b.two', change: { actions: ['update'], before: { x: 1 }, after: { x: 2 } } }, + { address: 'a.one', change: { actions: ['forget'], before: {}, after: null } }, + { address: 'c.three', change: { actions: ['delete', 'create'], before: {}, after: {}, after_unknown: { id: true } } } + ] + }) + assert.deepEqual(changes.map((change) => change.address), ['b.two', 'c.three']) + assert.deepEqual(summarize(changes), { create: 0, update: 1, delete: 0, replace: 1, 'no-op': 0, read: 0 }) +}) diff --git a/cloud/dev/scripts/classify-relay-production-capacity-director.mjs b/cloud/dev/scripts/classify-relay-production-capacity-director.mjs new file mode 100644 index 00000000000..e24bac7b6e3 --- /dev/null +++ b/cloud/dev/scripts/classify-relay-production-capacity-director.mjs @@ -0,0 +1,109 @@ +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' +import { isDeepStrictEqual } from 'node:util' + +function parseArguments(argv) { + if (argv.length !== 2 || argv[0] !== '--capacity-service-account' || !argv[1]) { + throw new Error('missing --capacity-service-account') + } + return argv[1] +} + +export function classifyProductionCapacityDirector(state, capacityServiceAccount) { + const currentIdentity = state.currentCapacityServiceAccount + if (currentIdentity !== null && currentIdentity !== capacityServiceAccount) { + throw new Error('director has an unexpected capacity identity') + } + const { + baseCells, + currentCells, + capacityCellIds, + targetCellId, + targetHardCap + } = state + if ( + !Array.isArray(baseCells) || + !Array.isArray(currentCells) || + !Array.isArray(capacityCellIds) || + ![600, 1000].includes(targetHardCap) || + new Set(capacityCellIds).size !== capacityCellIds.length || + !capacityCellIds.includes(targetCellId) || + baseCells.length !== currentCells.length || + new Set(baseCells.map((cell) => cell?.id)).size !== baseCells.length || + new Set(currentCells.map((cell) => cell?.id)).size !== currentCells.length + ) { + throw new Error('director topology transition input is invalid') + } + const capacityCells = new Set(capacityCellIds) + const normalizedCurrent = currentCells.map((current, index) => { + const base = baseCells[index] + if ( + typeof current?.id !== 'string' || + current.id !== base?.id + ) { + throw new Error('director topology cell identity is invalid') + } + if (!capacityCells.has(current.id)) { + if (!isDeepStrictEqual(current, base)) { + throw new Error('director topology changed outside the capacity rollout') + } + return current + } + if ( + base.connectionHardCap !== 1000 || + base.connectionUnobservedBound !== 60 || + ![600, 1000].includes(current.connectionHardCap) || + current.connectionUnobservedBound !== 60 + ) { + throw new Error('director capacity rollout state is invalid') + } + return { + ...current, + connectionHardCap: base.connectionHardCap, + connectionUnobservedBound: base.connectionUnobservedBound + } + }) + if ( + !isDeepStrictEqual(normalizedCurrent, baseCells) || + capacityCellIds.some((cellId) => !baseCells.some((cell) => cell.id === cellId)) + ) { + throw new Error('director topology is outside the reviewed capacity envelope') + } + const withTargetCap = (hardCap) => currentCells.map((cell) => + cell.id === targetCellId + ? { ...cell, connectionHardCap: hardCap, connectionUnobservedBound: 60 } + : cell + ) + const desiredCells = withTargetCap(targetHardCap) + const predecessorCells = withTargetCap(targetHardCap === 600 ? 1000 : 600) + const topologyPhase = isDeepStrictEqual(currentCells, desiredCells) + ? 'desired' + : isDeepStrictEqual(currentCells, predecessorCells) + ? 'predecessor' + : null + if (!topologyPhase) throw new Error('director topology is not a reviewed transition state') + return { + topologyPhase, + directorReady: + topologyPhase === 'desired' && currentIdentity === capacityServiceAccount, + desiredCells + } +} + +export function main(argv = process.argv.slice(2)) { + const capacityServiceAccount = parseArguments(argv) + const state = JSON.parse(readFileSync(0, 'utf8')) + process.stdout.write(`${JSON.stringify({ + event: 'relay_production_capacity_director_classified', + ...classifyProductionCapacityDirector(state, capacityServiceAccount) + })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + main() + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } +} diff --git a/cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs b/cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs new file mode 100644 index 00000000000..13b188ac042 --- /dev/null +++ b/cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + classifyProductionCapacityDirector +} from './classify-relay-production-capacity-director.mjs' + +const capacityServiceAccount = + 'orca-cloud-gha-relay-cap@onorca-cloud.iam.gserviceaccount.com' +const capacityCellIds = ['production-gce-c25', 'production-gce-c26'] +const baseCells = [ + { id: 'production-gce-c17', connectionHardCap: 600, connectionUnobservedBound: 60 }, + { id: 'production-gce-c25', connectionHardCap: 1000, connectionUnobservedBound: 60 }, + { id: 'production-gce-c26', connectionHardCap: 1000, connectionUnobservedBound: 60 } +] +const mixedCells = [ + baseCells[0], + { ...baseCells[1], connectionHardCap: 600 }, + baseCells[2] +] + +function state(overrides = {}) { + return { + baseCells, + currentCells: mixedCells, + capacityCellIds, + targetCellId: 'production-gce-c25', + targetHardCap: 1000, + currentCapacityServiceAccount: capacityServiceAccount, + ...overrides + } +} + +test('classifies one target while preserving completed rollout cells', () => { + assert.deepEqual( + classifyProductionCapacityDirector(state(), capacityServiceAccount), + { + topologyPhase: 'predecessor', + directorReady: false, + desiredCells: baseCells + } + ) +}) + +test('skips deployment only for exact topology and identity', () => { + assert.deepEqual( + classifyProductionCapacityDirector(state({ currentCells: baseCells }), capacityServiceAccount), + { topologyPhase: 'desired', directorReady: true, desiredCells: baseCells } + ) + assert.deepEqual( + classifyProductionCapacityDirector(state({ + currentCells: baseCells, + targetHardCap: 600 + }), capacityServiceAccount), + { + topologyPhase: 'predecessor', + directorReady: false, + desiredCells: mixedCells + } + ) +}) + +test('rejects an unknown topology or capacity identity', () => { + assert.throws( + () => classifyProductionCapacityDirector(state({ + currentCells: [baseCells[0], { ...baseCells[1], connectionHardCap: 700 }, baseCells[2]] + }), capacityServiceAccount), + /rollout state is invalid/ + ) + assert.throws( + () => classifyProductionCapacityDirector(state({ + currentCapacityServiceAccount: 'unexpected@onorca-cloud.iam.gserviceaccount.com' + }), capacityServiceAccount), + /unexpected capacity identity/ + ) + assert.throws( + () => classifyProductionCapacityDirector(state({ + currentCells: [{ ...baseCells[0], connectionHardCap: 1000 }, mixedCells[1], mixedCells[2]] + }), capacityServiceAccount), + /outside the capacity rollout/ + ) + assert.throws( + () => classifyProductionCapacityDirector(state({ + currentCells: [baseCells[0], { ...mixedCells[1], url: 'https://wrong.invalid' }, baseCells[2]] + }), capacityServiceAccount), + /outside the reviewed capacity envelope/ + ) + assert.throws( + () => classifyProductionCapacityDirector(state({ + targetCellId: 'production-gce-c17' + }), capacityServiceAccount), + /transition input is invalid/ + ) +}) diff --git a/cloud/dev/scripts/classify-relay-staging-bootstrap.mjs b/cloud/dev/scripts/classify-relay-staging-bootstrap.mjs new file mode 100644 index 00000000000..fd50b332c57 --- /dev/null +++ b/cloud/dev/scripts/classify-relay-staging-bootstrap.mjs @@ -0,0 +1,47 @@ +import { pathToFileURL } from 'node:url' + +export function classifyStagingBootstrap({ c2Kind, c2Admission, c3Kind, c3Admission }) { + for (const kind of [c2Kind, c3Kind]) { + if (!['legacy', 'modern'].includes(kind)) throw new Error('bootstrap runtime kind is invalid') + } + for (const admission of [c2Admission, c3Admission]) { + if (!['general', 'migration-only'].includes(admission)) { + throw new Error('bootstrap admission is not recoverable') + } + } + if (c2Kind === 'modern' && c3Kind === 'modern') return 'complete' + if (c2Kind === 'modern') return 'roll-c3' + if (c3Kind === 'modern') return 'roll-c2' + if (c2Admission === 'general') return 'normalize-and-roll-both' + if (c3Admission === 'general') return 'resume-c2-then-c3' + throw new Error('bootstrap has no general fallback') +} + +function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + return { + c2Kind: values['c2-kind'], + c2Admission: values['c2-admission'], + c3Kind: values['c3-kind'], + c3Admission: values['c3-admission'] + } +} + +export function main(argv = process.argv.slice(2)) { + process.stdout.write(`${classifyStagingBootstrap(parseArguments(argv))}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + main() + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } +} diff --git a/cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs b/cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs new file mode 100644 index 00000000000..66caa4fde4c --- /dev/null +++ b/cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { classifyStagingBootstrap } from './classify-relay-staging-bootstrap.mjs' + +const state = (c2Kind, c2Admission, c3Kind, c3Admission) => ({ + c2Kind, + c2Admission, + c3Kind, + c3Admission +}) + +test('classifies the fresh legacy bootstrap', () => { + assert.equal( + classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'general')), + 'normalize-and-roll-both' + ) +}) + +test('retries normalization after a failed C3 restart', () => { + assert.equal( + classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'migration-only')), + 'normalize-and-roll-both' + ) +}) + +test('resumes after C2 isolation', () => { + assert.equal( + classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'general')), + 'resume-c2-then-c3' + ) +}) + +test('resumes after C2 apply or a partial C3 transition', () => { + for (const c2Admission of ['general', 'migration-only']) { + for (const c3Admission of ['general', 'migration-only']) { + assert.equal( + classifyStagingBootstrap(state('modern', c2Admission, 'legacy', c3Admission)), + 'roll-c3' + ) + } + } +}) + +test('repairs an unexpected modern C3 before rolling legacy C2', () => { + assert.equal( + classifyStagingBootstrap(state('legacy', 'migration-only', 'modern', 'general')), + 'roll-c2' + ) +}) + +test('accepts already complete modern cells and rejects no-fallback legacy state', () => { + assert.equal( + classifyStagingBootstrap(state('modern', 'migration-only', 'modern', 'general')), + 'complete' + ) + assert.throws( + () => classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'migration-only')), + /no general fallback/ + ) +}) diff --git a/cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs b/cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs new file mode 100644 index 00000000000..76193746f2c --- /dev/null +++ b/cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs @@ -0,0 +1,305 @@ +// Derives, from workflow and script content, which workflows roll out against the shared Cloud SQL +// instance. Hand lists go stale silently; everything here is read back off disk. +import { readFileSync, readdirSync } from 'node:fs' +import { + RELAY_WORKFLOW_DIRECTORY, + RELAY_WORKFLOW_FILE_PREFIX, + relayWorkflowFile +} from './relay-repository.mjs' + +export const WORKFLOW_ROOT = RELAY_WORKFLOW_DIRECTORY +export const SCRIPT_ROOT = new URL('./', import.meta.url) + +export const LEASE_ACTION = './.github/actions/cloud-sql-rollout-lease' +export const PRODUCTION_LEASE = { + bucket: 'onorca-cloud-terraform-state', + object: 'terraform/state/cloud-sql-rollout/production.lock' +} +export const STAGING_LEASE = { + bucket: 'onorca-cloud-staging-terraform-state', + object: 'terraform/state/cloud-sql-rollout/staging.lock' +} + +export const PRODUCTION_GROUP = 'production-cloud-sql-rollout' +export const STAGING_GROUP = 'relay-staging-mutation' +export const SELECTABLE_GROUP = + "${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}" + +const selectable = (production, staging) => + `\${{ inputs.environment == 'production' && '${production}' || '${staging}' }}` + +export const SELECTABLE_LEASE = { + bucket: selectable(PRODUCTION_LEASE.bucket, STAGING_LEASE.bucket), + object: selectable(PRODUCTION_LEASE.object, STAGING_LEASE.object) +} + +export const LOCK_GROUPS = new Set([PRODUCTION_GROUP, STAGING_GROUP, SELECTABLE_GROUP]) + +export function readWorkflow(file) { + return readFileSync(new URL(file, WORKFLOW_ROOT), 'utf8') +} + +export function workflowFiles() { + return readdirSync(WORKFLOW_ROOT) + .filter((name) => name.endsWith('.yml') && name.startsWith(RELAY_WORKFLOW_FILE_PREFIX)) + .sort() +} + +// --- YAML-shaped readers (line based; the workflows are hand-written and uniformly indented) --- + +function indentOf(line) { + return line.length - line.trimStart().length +} + +function blockAfter(lines, index) { + const base = indentOf(lines[index]) + const body = [] + for (let i = index + 1; i < lines.length; i += 1) { + if (lines[i].trim() === '') { + body.push(lines[i]) + continue + } + if (indentOf(lines[i]) <= base) break + body.push(lines[i]) + } + return body +} + +export function concurrencyBlocks(text) { + const lines = text.split('\n') + const blocks = [] + lines.forEach((line, index) => { + if (line.trim() !== 'concurrency:') return + const body = blockAfter(lines, index) + blocks.push({ + group: body.find((l) => l.trim().startsWith('group:'))?.trim().slice('group:'.length).trim(), + cancelInProgress: body + .find((l) => l.trim().startsWith('cancel-in-progress:')) + ?.trim() + .slice('cancel-in-progress:'.length) + .trim() + }) + }) + return blocks +} + +export function jobs(text) { + const lines = text.split('\n') + const start = lines.findIndex((line) => line === 'jobs:') + if (start === -1) return [] + const found = [] + for (let i = start + 1; i < lines.length; i += 1) { + const match = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(lines[i]) + if (!match) continue + found.push({ id: match[1], start: i, body: blockAfter(lines, i) }) + } + return found.map((job) => ({ ...job, text: job.body.join('\n') })) +} + +function scalarField(jobText, key) { + const lines = jobText.split('\n') + const index = lines.findIndex((line) => /^ {4}[A-Za-z-]+:/.test(line) && line.trim().startsWith(`${key}:`)) + if (index === -1) return undefined + const inline = lines[index].trim().slice(`${key}:`.length).trim() + if (inline !== '' && inline !== '>-' && inline !== '|') return inline + return blockAfter(lines, index).join(' ').replace(/\s+/g, ' ').trim() +} + +export function jobNeeds(jobText) { + const raw = scalarField(jobText, 'needs') + if (!raw) return [] + return raw + .replace(/^\[|\]$/g, '') + .split(/[,\n]|\s+-\s+/) + .map((entry) => entry.replace(/^-/, '').trim()) + .filter(Boolean) +} + +export function jobIf(jobText) { + return scalarField(jobText, 'if') ?? '' +} + +export function leaseSteps(text) { + const lines = text.split('\n') + const steps = [] + lines.forEach((line, index) => { + if (line.trim() !== `- uses: ${LEASE_ACTION}`) return + const body = blockAfter(lines, index) + const read = (key) => + body.find((l) => l.trim().startsWith(`${key}:`))?.trim().slice(`${key}:`.length).trim() + steps.push({ + line: index + 1, + bucket: read('bucket'), + object: read('object'), + release: read('release') + }) + }) + return steps +} + +export function leaseStepsByJob(file) { + const text = readWorkflow(file) + const steps = leaseSteps(text) + return jobs(text).map((job) => ({ + id: job.id, + steps: steps.filter((step) => step.line > job.start + 1 && step.line <= job.start + 1 + job.body.length) + })) +} + +// --- trigger and reusable-call graph --- + +export function triggers(text) { + const lines = text.split('\n') + const index = lines.findIndex((line) => line === 'on:') + if (index === -1) return [] + return blockAfter(lines, index) + .map((line) => /^ {2}([a-z_]+):/.exec(line)?.[1]) + .filter(Boolean) +} + +export function isEntrypoint(text) { + return triggers(text).some((trigger) => trigger !== 'workflow_call') +} + +export function reusableCalls(text) { + const counts = new Map() + for (const match of text.matchAll(/uses: \.\/\.github\/workflows\/([A-Za-z0-9._-]+\.yml)/g)) { + counts.set(match[1], (counts.get(match[1]) ?? 0) + 1) + } + return counts +} + +export function entrypointsFor(file, seen = new Set()) { + if (seen.has(file)) return new Set() + seen.add(file) + if (isEntrypoint(readWorkflow(file))) return new Set([file]) + const reached = new Set() + for (const candidate of workflowFiles()) { + if (candidate === file) continue + if (!reusableCalls(readWorkflow(candidate)).has(file)) continue + for (const entry of entrypointsFor(candidate, seen)) reached.add(entry) + } + return reached +} + +// --- what counts as a Cloud SQL connection-budget rollout --- + +const COMMAND_PREFIX = /^(?:-\s+)?(?:run:\s*)?(?:[a-z_]+\s*=\s*"?\$\(\s*)?(?:if\s+|then\s+|else\s+|&&\s+|\|\|\s+|!\s+)*/ + +function commandLines(text) { + return text.split('\n').map((line) => line.trim().replace(COMMAND_PREFIX, '')) +} + +export function appliesTerraform(text) { + return commandLines(text).some((line) => /^terraform\b.*\bapply\b/.test(line)) +} + +export function runsCloudRunMutation(text) { + return commandLines(text).some((line) => + /^gcloud run (?:deploy\b|services (?:update|replace)\b|jobs (?:update|deploy)\b)/.test(line) + ) +} + +// Scripts that mint a Cloud Run revision, plus every script that re-exports one of them. +export function revisionMintingScripts() { + const self = new URL(import.meta.url).pathname.split('/').pop() + const names = readdirSync(SCRIPT_ROOT).filter( + (name) => name.endsWith('.mjs') && !name.endsWith('.test.mjs') && name !== self + ) + const source = new Map( + names.map((name) => [name, readFileSync(new URL(name, SCRIPT_ROOT), 'utf8')]) + ) + const minting = new Set( + names.filter((name) => { + const text = source.get(name) + return ( + text.includes("'--no-traffic'") || + /'run',\s*'services',\s*'update'/.test(text) || + /'run',\s*'deploy'/.test(text) + ) + }) + ) + for (let changed = true; changed; ) { + changed = false + for (const name of names) { + if (minting.has(name)) continue + const imports = [...source.get(name).matchAll(/from '\.\/([A-Za-z0-9._-]+\.mjs)'/g)].map( + (match) => match[1] + ) + if (!imports.some((imported) => minting.has(imported))) continue + minting.add(name) + changed = true + } + } + return minting +} + +export function mutatesSharedInstance(text, minters = revisionMintingScripts()) { + if (appliesTerraform(text)) return 'terraform apply against the reviewed relay cell templates' + if (runsCloudRunMutation(text)) return 'gcloud mints or replaces a Cloud Run revision' + for (const script of minters) { + if (text.includes(`dev/scripts/${script}`)) return `runs ${script}, which mints a Cloud Run revision` + } + return undefined +} + +// --- the declared contract --- + +const production = (extra = {}) => ({ env: 'production', group: PRODUCTION_GROUP, ...extra }) +const staging = (extra = {}) => ({ env: 'staging', group: STAGING_GROUP, ...extra }) +const eitherEnvironment = () => ({ env: 'selectable', group: SELECTABLE_GROUP }) + +// Keys are workflow filenames, which the public copy prefixes; the prefix lives in one place. +const named = (entries) => + Object.fromEntries( + entries.map(([file, entry]) => [ + relayWorkflowFile(file), + entry.leaseFiles + ? { ...entry, leaseFiles: entry.leaseFiles.map((member) => relayWorkflowFile(member)) } + : entry + ]) + ) + +export const LEASED_WORKFLOWS = named([ + ['deploy-relay-fence-broker.yml', production()], + ['deploy-relay-production.yml', production()], + ['deploy-relay-production-director.yml', production()], + ['deploy-relay-production-multi-target.yml', production()], + [ + 'deploy-relay-production-capacity.yml', + production({ + leaseFiles: ['deploy-relay-production-capacity-job.yml'], + reentrant: true + }) + ], + [ + 'deploy-relay-production-same-cap.yml', + production({ + leaseFiles: ['deploy-relay-production-same-cap-job.yml'], + reentrant: true + }) + ], + [ + 'operate-relay-production-rehome.yml', + production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] }) + ], + ['deploy-relay-asia-topology.yml', eitherEnvironment()], + ['operate-relay-asia-admission.yml', eitherEnvironment()], + ['deploy-relay-staging.yml', staging()], + ['deploy-relay-staging-gce-candidate.yml', staging()], + ['bootstrap-relay-staging-capacity.yml', staging()], + ['power-relay-staging.yml', staging()], + ['prove-relay-asia-staging.yml', staging()], + [ + 'prove-relay-staging-capacity.yml', + staging({ exclusiveBy: "inputs.mode == 'refresh-asia-c4-image'" }) + ], + ['recover-relay-staging-c4-image.yml', staging()] +]) + +export const NOT_A_CLOUD_SQL_CANDIDATE = named([ + [ + 'monitor-relay-production.yml', + 'Read-only. Its identity holds monitoring, logging, Cloud SQL and compute viewer roles only, and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget, so the durable lease would only let monitoring block a rollout and a rollout block monitoring.' + ] +]) diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs new file mode 100644 index 00000000000..88e4f8ccc60 --- /dev/null +++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs @@ -0,0 +1,1132 @@ +import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { pathToFileURL } from 'node:url' +import { isDeepStrictEqual } from 'node:util' + +const POLL_INTERVAL_MS = 5_000 +const MIGRATION_TIMEOUT_MS = 14 * 60 * 1000 +const CONNECTION_CAPACITY_PROTOCOL = 2 +export const DIRECTOR_REGIONAL_PLACEMENT_SECRET = + 'orca-cloud-relay-regional-placement-enabled' +export const DIRECTOR_REGIONAL_PLACEMENT_ENV = + 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' +export const DIRECTOR_REHOME_IDENTITY_ENV = + 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT' +export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE' +export const SELECTOR_ROLLBACK_TAG = 'selector-rollback' +export const SELECTOR_REVISION_MARKER = '3' +export const DIRECTOR_ADMISSION_ENVIRONMENT = Object.freeze({ + ORCA_RELAY_DATABASE_POOL_MAX: '3', + ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: '2', + ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX: '128', + ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS: '5', + ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS: '4000' +}) +const DIRECTOR_STARTUP_PROBE = + 'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1' + +export function taggedRevisionOrigin(serviceOrigin, tag) { + const url = new URL(serviceOrigin) + if ( + url.protocol !== 'https:' || + url.pathname !== '/' || + url.search || + url.hash || + !url.hostname.endsWith('.run.app') + ) { + throw new Error('Cloud Run service origin is not canonical') + } + if (!/^[a-z][a-z0-9-]{0,62}$/.test(tag)) throw new Error('invalid Cloud Run tag') + return `https://${tag}---${url.hostname}` +} + +export function activeRevision(service) { + const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0) + if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) { + throw new Error('relay service must have exactly one revision receiving 100% traffic') + } + return active[0].revisionName +} + +export function trafficTags(service) { + return (service.status?.traffic ?? []) + .map((entry) => entry.tag) + .filter((tag) => typeof tag === 'string') +} + +export function taggedTraffic(service, tag) { + const traffic = (service.status?.traffic ?? []).find((entry) => entry.tag === tag) + if (!traffic?.url || !traffic.revisionName) throw new Error(`Cloud Run tag ${tag} is not ready`) + return { origin: traffic.url, revision: traffic.revisionName } +} + +export function revisionEnvironment(revision) { + const entries = revision.spec?.containers?.[0]?.env ?? [] + return Object.fromEntries( + entries + .filter((entry) => entry.name && 'value' in entry) + .map((entry) => [entry.name, entry.value]) + ) +} + +export function revisionSecretEnvironment(revision) { + const entries = revision.spec?.containers?.[0]?.env ?? [] + return Object.fromEntries( + entries + .map((entry) => { + const reference = entry.valueSource?.secretKeyRef ?? entry.valueFrom?.secretKeyRef + return [entry.name, reference && { + secret: reference.secret ?? reference.name, + version: reference.version ?? reference.key + }] + }) + .filter(([name, reference]) => name && reference) + ) +} + +export function revisionMinimumInstances(revision) { + return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0) +} + +export function revisionMaximumInstances(revision) { + const value = Number(revision.metadata?.annotations?.['autoscaling.knative.dev/maxScale']) + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error('serving revision has no bounded maximum instance count') + } + return value +} + +function hasExpectedEnvironment(environment, expected) { + return Object.entries(expected).every(([key, value]) => environment[key] === value) +} + +function projectServiceAccount(config, argument) { + const value = config[argument] + if (value === undefined) return undefined + const suffix = `@${config.project}.iam.gserviceaccount.com` + const account = value.endsWith(suffix) ? value.slice(0, -suffix.length) : '' + if (!/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/.test(account)) { + throw new Error(`--${argument} must belong to the selected project`) + } + return value +} + +function directorCellsJson(value, { allowMissingRegion = false } = {}) { + if (value === undefined) return undefined + if (value.length > 100_000 || /[\r\n]/.test(value)) { + throw new Error('--director-cells-json is invalid') + } + const cells = JSON.parse(value) + if (!Array.isArray(cells) || cells.length < 1 || cells.length > 100) { + throw new Error('--director-cells-json must contain 1..100 cells') + } + const ids = new Set() + for (const cell of cells) { + const keys = Object.keys(cell ?? {}).sort() + const expectedKeys = [ + 'capacityRequests', + 'id', + 'initiallyEnabled', + ...(allowMissingRegion && cell?.region === undefined ? [] : ['region']), + 'url', + ...(cell?.connectionHardCap === undefined + ? [] + : ['connectionHardCap', 'connectionUnobservedBound']) + ].sort() + let origin + try { + origin = new URL(cell?.url) + } catch { + throw new Error('--director-cells-json contains an invalid cell URL') + } + const cap = cell?.connectionHardCap + const bound = cell?.connectionUnobservedBound + if ( + JSON.stringify(keys) !== JSON.stringify(expectedKeys) || + !/^[a-z][a-z0-9-]{0,39}$/.test(cell.id ?? '') || + ids.has(cell.id) || + origin.protocol !== 'https:' || + origin.origin !== cell.url || + !Number.isSafeInteger(cell.capacityRequests) || + cell.capacityRequests < 1 || + !['us-central1', 'asia-east2'].includes( + cell.region ?? (allowMissingRegion ? 'us-central1' : undefined) + ) || + typeof cell.initiallyEnabled !== 'boolean' || + (cap !== undefined && + (![600, 1_000, 3_000].includes(cap) || + !Number.isSafeInteger(bound) || + bound < 0 || + bound >= cap - 100)) + ) { + throw new Error('--director-cells-json contains an invalid cell') + } + ids.add(cell.id) + } + return JSON.stringify( + cells.map((cell) => ({ + id: cell.id, + url: cell.url, + capacityRequests: cell.capacityRequests, + region: cell.region ?? 'us-central1', + initiallyEnabled: cell.initiallyEnabled, + ...(cell.connectionHardCap === undefined + ? {} + : { + connectionHardCap: cell.connectionHardCap, + connectionUnobservedBound: cell.connectionUnobservedBound + }) + })) + ) +} + +export function directorTopologyChange(currentValue, desiredValue, cellId) { + const current = JSON.parse(directorCellsJson(currentValue, { allowMissingRegion: true })) + const desired = JSON.parse(directorCellsJson(desiredValue)) + if ( + current.length !== desired.length || + desired.some(({ id }, index) => id !== current[index]?.id) + ) { + throw new Error('director topology changes the Relay cell set or order') + } + let changed = false + for (let index = 0; index < desired.length; index += 1) { + const before = structuredClone(current[index]) + const after = structuredClone(desired[index]) + if (after.id === cellId) { + delete before.connectionHardCap + delete before.connectionUnobservedBound + delete after.connectionHardCap + delete after.connectionUnobservedBound + changed = !isDeepStrictEqual(current[index], desired[index]) + } + if (!isDeepStrictEqual(before, after)) { + throw new Error('director topology changes fields outside the reviewed capacity pair') + } + } + if (!desired.some(({ id }) => id === cellId)) { + throw new Error('director topology omits the reviewed capacity cell') + } + return { changed, value: JSON.stringify(desired) } +} + +export function directorCellSetAddition(currentValue, desiredValue) { + const current = JSON.parse(directorCellsJson(currentValue, { allowMissingRegion: true })) + const desired = JSON.parse(directorCellsJson(desiredValue)) + if (desired.length < current.length) { + throw new Error('director topology addition cannot remove cells') + } + const desiredById = new Map(desired.map((cell) => [cell.id, cell])) + if (current.some((cell) => !isDeepStrictEqual(desiredById.get(cell.id), cell))) { + throw new Error('director topology addition changes an existing cell') + } + const currentIds = new Set(current.map((cell) => cell.id)) + const additions = desired.filter((cell) => !currentIds.has(cell.id)) + if (additions.some((cell) => cell.initiallyEnabled !== false)) { + throw new Error('director topology additions must start disabled') + } + return { changed: additions.length > 0, value: JSON.stringify(desired) } +} + +export function directorDeploymentEnvironment(config) { + const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1] + if (config.image !== undefined && imageDigest === undefined) { + throw new Error('--image must use an immutable digest for director deployments') + } + const environment = { + ...DIRECTOR_ADMISSION_ENVIRONMENT, + ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER, + ...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest }) + } + const serviceAccount = projectServiceAccount(config, 'capacity-service-account') + const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account') + const rehomeDirectorServiceAccount = projectServiceAccount( + config, + 'rehome-director-service-account' + ) + const cellsJson = directorCellsJson(config['director-cells-json']) + if (serviceAccount !== undefined) { + environment.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT = serviceAccount + } + if (asiaProofServiceAccount !== undefined) { + environment.ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT = asiaProofServiceAccount + } + if (rehomeDirectorServiceAccount !== undefined) { + environment[DIRECTOR_REHOME_IDENTITY_ENV] = rehomeDirectorServiceAccount + environment[DIRECTOR_REHOME_AUDIENCE_ENV] = config['rehome-audience'] + } + if (cellsJson !== undefined) environment.ORCA_RELAY_CELLS_JSON = cellsJson + return environment +} + +export function environmentUpdateValue(environment) { + const entries = Object.entries(environment) + if (entries.every(([key, value]) => !key.includes(',') && !value.includes(','))) { + return entries.map(([key, value]) => `${key}=${value}`).join(',') + } + const delimiter = ['~', '|', '@', '%', ';'].find((candidate) => + entries.every(([key, value]) => !key.includes(candidate) && !value.includes(candidate)) + ) + if (!delimiter) throw new Error('candidate environment has no safe gcloud delimiter') + return `^${delimiter}^${entries.map(([key, value]) => `${key}=${value}`).join(delimiter)}` +} + +export function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) + values[key.slice(2)] = value + } + const required = ['project', 'region', 'service', 'image', 'role', 'release-id'] + for (const key of required) if (!values[key]) throw new Error(`missing --${key}`) + if (!['director', 'cell'].includes(values.role)) throw new Error('--role must be director or cell') + if (values['min-instances'] !== undefined && !/^(0|[1-9][0-9]*)$/.test(values['min-instances'])) { + throw new Error('--min-instances must be a nonnegative integer') + } + if (values['max-instances'] !== undefined && !/^[1-9][0-9]*$/.test(values['max-instances'])) { + throw new Error('--max-instances must be a positive integer') + } + if (values.role === 'cell') { + for (const key of ['director-origin', 'admin-audience']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (!['enabled', 'disabled'].includes(values['final-admission'] ?? 'enabled')) { + throw new Error('--final-admission must be enabled or disabled') + } + if ( + values['capacity-service-account'] !== undefined || + values['director-cells-json'] !== undefined || + values['runtime-service-account'] !== undefined || + values['rehome-director-service-account'] !== undefined || + values['rehome-audience'] !== undefined || + values['expected-rehome-generation'] !== undefined || + values['rehome-control-origin'] !== undefined + ) { + throw new Error('director configuration arguments require --role director') + } + } + if ( + values.role === 'director' && + values['capacity-cell-id'] !== undefined && + values['director-cells-json'] === undefined + ) { + throw new Error('--capacity-cell-id requires --director-cells-json') + } + if (values['regional-placement-enabled'] !== undefined) { + throw new Error('regional placement changes use the audited runtime-setting step') + } + if ( + values['regional-placement-secret-version'] !== undefined && + !/^[1-9][0-9]*$/.test(values['regional-placement-secret-version']) + ) { + throw new Error('--regional-placement-secret-version must be a positive integer') + } + if (!['true', 'false'].includes(values['prune-revisions'] ?? 'false')) { + throw new Error('--prune-revisions must be true or false') + } + if (!['true', 'false'].includes(values['bootstrap-runtime-identity'] ?? 'false')) { + throw new Error('--bootstrap-runtime-identity must be true or false') + } + if (values.role === 'director') { + directorDeploymentEnvironment(values) + projectServiceAccount(values, 'runtime-service-account') + projectServiceAccount(values, 'predecessor-runtime-service-account') + if ( + values['bootstrap-runtime-identity'] === 'true' && + (!values['runtime-service-account'] || + !values['predecessor-runtime-service-account'] || + !values['predecessor-image-digest']) + ) { + throw new Error('runtime identity bootstrap requires exact predecessor digest and identities') + } + if ( + values['predecessor-image-digest'] !== undefined && + !/^sha256:[a-f0-9]{64}$/.test(values['predecessor-image-digest']) + ) { + throw new Error('--predecessor-image-digest must be an immutable digest') + } + const rehomePair = [ + 'rehome-director-service-account', + 'rehome-audience' + ].map((key) => values[key] !== undefined) + if (rehomePair[0] !== rehomePair[1]) { + throw new Error('rehome identity and audience must be configured together') + } + if (values['rehome-audience'] !== undefined) { + const audience = new URL(values['rehome-audience']) + if ( + audience.protocol !== 'https:' || + audience.pathname !== '/v1/admin/host-drain' || + audience.search || + audience.hash + ) { + throw new Error('--rehome-audience must be an exact host-drain HTTPS URL') + } + } + const controlArguments = [ + 'expected-rehome-generation', + 'rehome-control-origin', + 'admin-audience' + ].map((key) => values[key] !== undefined) + if (controlArguments.some(Boolean) && !controlArguments.every(Boolean)) { + throw new Error('durable rehome verification arguments must be configured together') + } + if ( + values['expected-rehome-generation'] !== undefined && + !/^(0|[1-9][0-9]*)$/.test(values['expected-rehome-generation']) + ) { + throw new Error('--expected-rehome-generation must be a nonnegative integer') + } + if (values['rehome-control-origin'] !== undefined) { + const origin = new URL(values['rehome-control-origin']) + if (origin.protocol !== 'https:' || origin.origin !== values['rehome-control-origin']) { + throw new Error('--rehome-control-origin must be an HTTPS origin') + } + } + } + return values +} + +function commandJson(args) { + const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) + if (result.status !== 0) { + throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`) + } + return JSON.parse(result.stdout) +} + +function commandText(args, { sensitive = false } = {}) { + const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) + if (result.status !== 0) { + const detail = sensitive ? 'credential command failed' : result.stderr.trim() + throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${detail}`) + } + return result.stdout.trim() +} + +export function suppliedAdminIdentityToken(environment = process.env) { + const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN + if (token === undefined) return null + // The workflow supplies a masked Google ID token because external-account gcloud cannot mint one directly. + if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) { + throw new Error('invalid supplied admin identity token') + } + return token +} + +function adminIdentityToken(config) { + return ( + suppliedAdminIdentityToken() ?? + commandText(['auth', 'print-identity-token', `--audiences=${config['admin-audience']}`], { + sensitive: true + }) + ) +} + +function serviceArguments(config) { + return ['--project', config.project, '--region', config.region] +} + +export function directorStartupProbeArguments(role) { + return role === 'director' ? ['--startup-probe', DIRECTOR_STARTUP_PROBE] : [] +} + +function describeService(config) { + return commandJson([ + 'run', + 'services', + 'describe', + config.service, + ...serviceArguments(config), + '--format=json' + ]) +} + +function describeRevision(config, revision) { + return commandJson([ + 'run', + 'revisions', + 'describe', + revision, + ...serviceArguments(config), + '--format=json' + ]) +} + +function listRevisions(config) { + return commandJson([ + 'run', + 'revisions', + 'list', + '--service', + config.service, + ...serviceArguments(config), + '--format=json' + ]) +} + +function deleteRevision(config, revision) { + commandText([ + 'run', + 'revisions', + 'delete', + revision, + ...serviceArguments(config), + '--quiet' + ]) +} + +function updateTraffic(config, args) { + commandText([ + 'run', + 'services', + 'update-traffic', + config.service, + ...serviceArguments(config), + ...args, + '--quiet' + ]) +} + +function removeDirectorTrafficTags(config, operations, retained = new Set()) { + const tags = trafficTags(operations.describeService(config)).filter( + (tag) => !retained.has(tag) + ) + if (tags.length === 0) return + try { + operations.updateTraffic(config, [`--remove-tags=${tags.join(',')}`]) + } catch (error) { + const remaining = new Set(trafficTags(operations.describeService(config))) + if (tags.some((tag) => remaining.has(tag))) throw error + } +} + +function pruneDirectorRevisions(config, operations) { + const service = operations.describeService(config) + const retained = new Set([ + activeRevision(service), + taggedTraffic(service, SELECTOR_ROLLBACK_TAG).revision + ]) + for (const revision of operations.listRevisions(config)) { + const name = revision.metadata?.name + if (!name) throw new Error('director revision list contains an unnamed revision') + if (!retained.has(name)) operations.deleteRevision(config, name) + } + const remaining = operations + .listRevisions(config) + .map((revision) => revision.metadata?.name) + if ( + remaining.length !== retained.size || + remaining.some((name) => !name || !retained.has(name)) + ) { + throw new Error('old director revisions remain after deployment') + } +} + +function deployCandidate( + config, + tag, + env = {}, + image = config.image, + minInstances = config['min-instances'], + maxInstances, + regionalPlacementVersion +) { + const args = [ + 'run', + 'services', + 'update', + config.service, + ...serviceArguments(config), + '--image', + image, + '--tag', + tag, + '--no-traffic' + ] + if (maxInstances !== undefined) args.push('--max', String(maxInstances)) + if (config.role === 'director') { + args.push( + '--update-secrets', + `${DIRECTOR_REGIONAL_PLACEMENT_ENV}=${DIRECTOR_REGIONAL_PLACEMENT_SECRET}:${regionalPlacementVersion}` + ) + if (config['runtime-service-account'] !== undefined) { + args.push('--service-account', config['runtime-service-account']) + } + } + args.push(...directorStartupProbeArguments(config.role)) + if (minInstances !== undefined) { + args.push('--min-instances', String(minInstances)) + } + const entries = Object.entries(env) + if (entries.length > 0) { + args.push('--update-env-vars', environmentUpdateValue(env)) + } + args.push('--quiet') + commandText(args) +} + +function revisionShape(revision, mutableEnvironment, allowServiceAccountChange = false) { + const spec = structuredClone(revision.spec ?? {}) + if (allowServiceAccountChange) delete spec.serviceAccountName + for (const container of spec.containers ?? []) { + delete container.image + delete container.startupProbe + container.env = (container.env ?? []).filter( + ({ name }) => !(name in mutableEnvironment) + ) + } + const ignoredAnnotations = new Set([ + 'autoscaling.knative.dev/minScale', + 'run.googleapis.com/client-name', + 'run.googleapis.com/client-version', + 'run.googleapis.com/operation-id', + 'serving.knative.dev/creator' + ]) + const annotations = Object.fromEntries( + Object.entries(revision.metadata?.annotations ?? {}).filter( + ([name]) => !ignoredAnnotations.has(name) + ) + ) + return { spec, annotations } +} + +function assertPreservedRevisionShape( + serving, + candidate, + mutableEnvironment, + allowServiceAccountChange = false +) { + if (!isDeepStrictEqual( + revisionShape(serving, mutableEnvironment, allowServiceAccountChange), + revisionShape(candidate, mutableEnvironment, allowServiceAccountChange) + )) { + throw new Error('director candidate changed unrelated revision shape') + } +} + +function releaseLabel(releaseId) { + const normalized = releaseId.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') + if (!normalized) throw new Error('release id has no usable characters') + return normalized.slice(0, 32).replace(/-$/g, '') +} + +export function cloudRunTrafficTag(service, prefix, releaseId) { + if (!/^[a-z][a-z0-9-]*$/.test(service)) throw new Error('invalid Cloud Run service name') + if (!/^[a-z][a-z0-9-]*$/.test(prefix)) throw new Error('invalid Cloud Run tag prefix') + const digest = createHash('sha256').update(releaseLabel(releaseId)).digest('hex').slice(0, 9) + const tag = `${prefix}-${digest}` + // Cloud Run imposes this combined bound in addition to the standalone tag bound. + if (service.length + tag.length > 46) throw new Error('Cloud Run service leaves no safe tag space') + return tag +} + +function cellIdentifier(sourceCellId, releaseId) { + const suffix = releaseLabel(releaseId) + const available = 128 - suffix.length - 2 + return `${sourceCellId.slice(0, available)}--${suffix}` +} + +async function waitForHealth(origin, connectionCapacityProtocol) { + const response = await fetch(`${origin}/health`, { signal: AbortSignal.timeout(15_000) }) + const body = await response.json() + if ( + !response.ok || + body.ok !== true || + (connectionCapacityProtocol !== undefined && + body.connectionCapacityProtocol !== connectionCapacityProtocol) + ) { + throw new Error(`candidate health failed at ${origin}`) + } +} + +export async function waitForEvacuationCapacity( + adminPost, + sourceCellId, + targetCellId, + { pollIntervalMs = POLL_INTERVAL_MS, timeoutMs = MIGRATION_TIMEOUT_MS } = {} +) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + return await adminPost('/v1/admin/evacuation-capacity', { + v: 1, + sourceCellId, + targetCellId + }) + } catch (error) { + if (!(error instanceof Error) || !error.message.endsWith(': target_cell_unavailable')) throw error + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)) + } + } + throw new Error('timed out waiting for candidate cell readiness') +} + +async function adminClient(config) { + const token = adminIdentityToken(config) + return async (path, body) => { + const response = await fetch(`${config['director-origin']}${path}`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }) + const result = await response.json().catch(() => ({ error: `http_${response.status}` })) + if (!response.ok) throw new Error(`${path} failed: ${result.error ?? response.status}`) + return result + } +} + +export async function assertRegionalRehomeDisabled( + config, + origin, + fetchImpl = fetch +) { + const response = await fetchImpl(`${origin}/v1/admin/regional-rehome-control`, { + method: 'POST', + headers: { + authorization: `Bearer ${adminIdentityToken(config)}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, action: 'inspect' }), + signal: AbortSignal.timeout(30_000) + }) + const result = await response.json().catch(() => ({ error: `http_${response.status}` })) + if (!response.ok) { + throw new Error(`regional rehome inspection failed: ${result.error ?? response.status}`) + } + const generation = Number(config['expected-rehome-generation']) + if ( + result.v !== 1 || + result.control?.enabled !== false || + result.control?.generation !== generation + ) { + throw new Error('regional rehome control is not durably disabled at the expected generation') + } + return result.control +} + +function assertDirectorRevisionIdentity(revision, config) { + const expectedRuntimeServiceAccount = config['runtime-service-account'] + if ( + expectedRuntimeServiceAccount !== undefined && + revision.spec?.serviceAccountName !== expectedRuntimeServiceAccount + ) { + throw new Error('director revision uses an unexpected runtime service account') + } +} + +export async function deployDirector(config, tag, overrides = {}) { + const operations = { + deployCandidate, + describeService, + describeRevision, + listRevisions, + deleteRevision, + updateTraffic, + waitForHealth, + assertRegionalRehomeDisabled, + ...overrides + } + // Why: gcloud does not carry minScale onto a new revision, and the candidate below takes + // 100% of traffic. Inheriting the serving floor keeps one owner for the number instead of + // restating it here; public admission is per-instance, so losing it shrinks fleet capacity. + const initialService = operations.describeService(config) + const servingRevision = operations.describeRevision(config, activeRevision(initialService)) + const bootstrapRuntimeIdentity = config['bootstrap-runtime-identity'] === 'true' + if (bootstrapRuntimeIdentity) { + if ( + servingRevision.spec?.serviceAccountName !== + config['predecessor-runtime-service-account'] + ) { + throw new Error('director predecessor runtime service account does not match') + } + if ( + config['predecessor-runtime-service-account'] === config['runtime-service-account'] + ) { + throw new Error('director runtime identity bootstrap has already completed') + } + const predecessorDigest = servingRevision.spec?.containers?.[0]?.image?.split('@').at(-1) + if (predecessorDigest !== config['predecessor-image-digest']) { + throw new Error('director predecessor image digest does not match') + } + } else { + assertDirectorRevisionIdentity(servingRevision, config) + } + const servingMinimumInstances = revisionMinimumInstances(servingRevision) + const servingMaximumInstances = revisionMaximumInstances(servingRevision) + const requiredMaximumInstances = config['max-instances'] === undefined + ? servingMaximumInstances + : Number(config['max-instances']) + if (servingMaximumInstances !== requiredMaximumInstances) { + throw new Error( + `serving revision holds ${servingMaximumInstances} maximum instances, expected ${requiredMaximumInstances}` + ) + } + const servingSecrets = revisionSecretEnvironment(servingRevision) + const servingRegionalPlacementVersion = + servingSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version + const targetRegionalPlacementVersion = + config['regional-placement-secret-version'] ?? servingRegionalPlacementVersion + if (!/^[1-9][0-9]*$/.test(targetRegionalPlacementVersion ?? '')) { + throw new Error('director deployment requires an exact regional placement secret version') + } + const rollbackRegionalPlacementVersion = + /^[1-9][0-9]*$/.test(servingRegionalPlacementVersion ?? '') + ? servingRegionalPlacementVersion + : targetRegionalPlacementVersion + const requiredMinimumInstances = + config['min-instances'] === undefined + ? servingMinimumInstances + : Number(config['min-instances']) + const requiredCapacityProtocol = + config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined + const currentEnvironment = revisionEnvironment(servingRevision) + const deploymentEnvironment = directorDeploymentEnvironment(config) + const mutableEnvironment = { + ...deploymentEnvironment, + [DIRECTOR_REGIONAL_PLACEMENT_ENV]: '' + } + const topology = config['director-cells-json'] === undefined + ? { changed: false } + : config['capacity-cell-id'] === undefined + ? directorCellSetAddition( + currentEnvironment.ORCA_RELAY_CELLS_JSON, + deploymentEnvironment.ORCA_RELAY_CELLS_JSON + ) + : directorTopologyChange( + currentEnvironment.ORCA_RELAY_CELLS_JSON, + deploymentEnvironment.ORCA_RELAY_CELLS_JSON, + config['capacity-cell-id'] + ) + const verifyRehomeDisabled = async (origin) => { + if (config['expected-rehome-generation'] === undefined) return + await operations.assertRegionalRehomeDisabled(config, origin) + } + if (!bootstrapRuntimeIdentity) { + await verifyRehomeDisabled(config['rehome-control-origin']) + } + removeDirectorTrafficTags(config, operations) + let deployed + let promoted = false + try { + operations.deployCandidate( + config, + SELECTOR_ROLLBACK_TAG, + deploymentEnvironment, + config.image, + 0, + requiredMaximumInstances, + rollbackRegionalPlacementVersion + ) + const rollback = taggedTraffic( + operations.describeService(config), + SELECTOR_ROLLBACK_TAG + ) + const rollbackRevision = operations.describeRevision(config, rollback.revision) + assertDirectorRevisionIdentity(rollbackRevision, config) + assertPreservedRevisionShape( + servingRevision, + rollbackRevision, + mutableEnvironment, + bootstrapRuntimeIdentity + ) + const rollbackEnvironment = revisionEnvironment(rollbackRevision) + const rollbackSecrets = revisionSecretEnvironment(rollbackRevision) + if ( + rollbackEnvironment.ORCA_RELAY_ROLE !== 'director' || + rollbackEnvironment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== + SELECTOR_REVISION_MARKER || + !hasExpectedEnvironment(rollbackEnvironment, deploymentEnvironment) || + rollbackSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.secret !== + DIRECTOR_REGIONAL_PLACEMENT_SECRET || + rollbackSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version !== + rollbackRegionalPlacementVersion || + revisionMinimumInstances(rollbackRevision) !== 0 + ) { + throw new Error('rollback revision is not selector-compatible') + } + await operations.waitForHealth(rollback.origin, requiredCapacityProtocol) + await verifyRehomeDisabled(rollback.origin) + operations.deployCandidate( + config, + tag, + deploymentEnvironment, + config.image, + requiredMinimumInstances, + requiredMaximumInstances, + targetRegionalPlacementVersion + ) + const candidate = taggedTraffic(operations.describeService(config), tag) + const candidateRevision = operations.describeRevision(config, candidate.revision) + assertDirectorRevisionIdentity(candidateRevision, config) + assertPreservedRevisionShape( + servingRevision, + candidateRevision, + mutableEnvironment, + bootstrapRuntimeIdentity + ) + const environment = revisionEnvironment(candidateRevision) + const secrets = revisionSecretEnvironment(candidateRevision) + if ( + environment.ORCA_RELAY_ROLE !== 'director' || + environment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== SELECTOR_REVISION_MARKER || + !hasExpectedEnvironment(environment, deploymentEnvironment) || + secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.secret !== + DIRECTOR_REGIONAL_PLACEMENT_SECRET || + secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version !== + targetRegionalPlacementVersion + ) { + throw new Error('stable relay service is not selector-compatible') + } + // Fail before the traffic move, so a candidate that lost the floor never serves. + const candidateMinimumInstances = revisionMinimumInstances(candidateRevision) + if (candidateMinimumInstances !== requiredMinimumInstances) { + throw new Error( + `candidate holds ${candidateMinimumInstances} minimum instances, expected ${requiredMinimumInstances}` + ) + } + await operations.waitForHealth(candidate.origin, requiredCapacityProtocol) + await verifyRehomeDisabled(candidate.origin) + operations.updateTraffic(config, [`--to-tags=${tag}=100`]) + promoted = true + removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG])) + deployed = { + event: 'director_deployed', + revision: candidate.revision, + rollbackRevision: rollback.revision, + topologyChanged: topology.changed + } + } catch (error) { + const recoveryErrors = [error] + if (promoted) { + try { + operations.updateTraffic(config, [`--to-tags=${SELECTOR_ROLLBACK_TAG}=100`]) + } catch (rollbackError) { + recoveryErrors.push(rollbackError) + } + } + try { + removeDirectorTrafficTags( + config, + operations, + promoted ? new Set([SELECTOR_ROLLBACK_TAG]) : new Set() + ) + } catch (cleanupError) { + recoveryErrors.push(cleanupError) + } + if (recoveryErrors.length > 1) { + const details = recoveryErrors + .map((failure) => failure instanceof Error ? failure.message : String(failure)) + .join('; ') + throw new AggregateError(recoveryErrors, `director deploy recovery failed: ${details}`) + } + throw error + } + if (config['prune-revisions'] === 'true') pruneDirectorRevisions(config, operations) + process.stdout.write(`${JSON.stringify(deployed)}\n`) +} + +async function waitForTargetRegistration(adminPost, sourceCellId, targetCellId) { + const deadline = Date.now() + MIGRATION_TIMEOUT_MS + while (Date.now() < deadline) { + const status = await adminPost('/v1/admin/evacuation-status', { + v: 1, + sourceCellId, + targetCellId, + completeReady: false + }) + process.stdout.write( + `${JSON.stringify({ event: 'migration_registration', ...status })}\n` + ) + if (status.inProgress === status.targetRegistered) return + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) + } + throw new Error('timed out waiting for target control registrations') +} + +async function waitForCompletion(adminPost, sourceCellId, targetCellId) { + const deadline = Date.now() + MIGRATION_TIMEOUT_MS + while (Date.now() < deadline) { + const status = await adminPost('/v1/admin/evacuation-status', { + v: 1, + sourceCellId, + targetCellId, + completeReady: true + }) + process.stdout.write(`${JSON.stringify({ event: 'migration_completion', ...status })}\n`) + if (status.inProgress === 0) return + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) + } + throw new Error('timed out waiting for source activity to drain') +} + +async function startAllEvacuations(adminPost, sourceCellId, targetCellId) { + for (;;) { + const result = await adminPost('/v1/admin/evacuate-cell', { + v: 1, + sourceCellId, + targetCellId, + limit: 100 + }) + process.stdout.write(`${JSON.stringify({ event: 'migration_batch', started: result.started })}\n`) + if (result.started === 0) return + } +} + +async function deployCell(config, tag, oldTag, drainTag) { + const initialService = describeService(config) + const currentRevision = activeRevision(initialService) + const currentRevisionState = describeRevision(config, currentRevision) + const currentEnv = revisionEnvironment(currentRevisionState) + const currentImage = currentRevisionState.spec?.containers?.[0]?.image + const sourceCellId = currentEnv.ORCA_RELAY_CELL_ID + const sourceOrigin = currentEnv.ORCA_RELAY_CELL_URL + const capacityRequests = Number(currentEnv.ORCA_RELAY_CELL_CAPACITY) + if ( + currentEnv.ORCA_RELAY_ROLE !== 'cell' || + !sourceCellId || + !sourceOrigin || + !currentImage || + !Number.isInteger(capacityRequests) || + capacityRequests <= 0 + ) { + throw new Error('active cell revision has invalid role, identity, image, or capacity') + } + updateTraffic(config, [`--update-tags=${drainTag}=${currentRevision}`]) + const drainRevision = taggedTraffic(describeService(config), drainTag) + const previousOrigin = taggedRevisionOrigin(initialService.status.url, oldTag) + const candidateOrigin = taggedRevisionOrigin(initialService.status.url, tag) + const targetCellId = cellIdentifier(sourceCellId, config['release-id']) + const adminPost = await adminClient(config) + + await adminPost('/v1/admin/cell-config', { + v: 1, + cellId: targetCellId, + cellUrl: candidateOrigin, + capacityRequests, + enabled: false + }) + deployCandidate(config, tag, { + ORCA_RELAY_CELL_ID: targetCellId, + ORCA_RELAY_CELL_URL: candidateOrigin, + ORCA_RELAY_PUBLIC_URL: candidateOrigin + }) + const candidate = taggedTraffic(describeService(config), tag) + if (candidate.origin !== candidateOrigin) { + throw new Error('queried candidate tag URL mismatches its configured origin') + } + await waitForHealth(candidate.origin) + deployCandidate( + config, + oldTag, + { + ORCA_RELAY_CELL_ID: sourceCellId, + ORCA_RELAY_CELL_URL: previousOrigin, + ORCA_RELAY_PUBLIC_URL: previousOrigin + }, + currentImage + ) + const previous = taggedTraffic(describeService(config), oldTag) + if (previous.origin !== previousOrigin) { + throw new Error('queried previous tag URL mismatches its configured origin') + } + await waitForHealth(previous.origin) + + // HTTP health precedes the authenticated heartbeat that makes a migration target eligible. + await waitForEvacuationCapacity(adminPost, sourceCellId, targetCellId) + await adminPost('/v1/admin/cell-state', { v: 1, cellId: sourceCellId, enabled: false }) + let sourceReconfigured = false + try { + const capacity = await adminPost('/v1/admin/evacuation-capacity', { + v: 1, + sourceCellId, + targetCellId + }) + process.stdout.write(`${JSON.stringify({ event: 'migration_capacity', ...capacity })}\n`) + if (capacity.requiredTargetUnits > capacity.availableTargetUnits) { + throw new Error('candidate lacks durable reservation headroom for target-first migration') + } + // The keeper uses the old image and cell identity without competing with live controls. + await adminPost('/v1/admin/cell-config', { + v: 1, + cellId: sourceCellId, + cellUrl: previous.origin, + capacityRequests, + enabled: false + }) + sourceReconfigured = true + await adminPost('/v1/admin/cell-config', { + v: 1, + cellId: targetCellId, + cellUrl: candidate.origin, + capacityRequests, + enabled: true + }) + } catch (error) { + if (sourceReconfigured) { + await adminPost('/v1/admin/cell-config', { + v: 1, + cellId: sourceCellId, + cellUrl: sourceOrigin, + capacityRequests, + enabled: true + }) + } else { + await adminPost('/v1/admin/cell-state', { v: 1, cellId: sourceCellId, enabled: true }) + } + throw error + } + await startAllEvacuations(adminPost, sourceCellId, targetCellId) + + await fetch(`${drainRevision.origin}/v1/admin/drain`, { + method: 'POST', + headers: { + authorization: `Bearer ${adminIdentityToken(config)}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, graceMs: 120_000 }), + signal: AbortSignal.timeout(30_000) + }).then(async (response) => { + if (!response.ok) throw new Error(`old revision drain failed: ${response.status}`) + }) + + await waitForTargetRegistration(adminPost, sourceCellId, targetCellId) + updateTraffic(config, [`--to-tags=${tag}=100`]) + await waitForCompletion(adminPost, sourceCellId, targetCellId) + const finalEnabled = (config['final-admission'] ?? 'enabled') === 'enabled' + if (!finalEnabled) { + // GCE-backed staging keeps stamped cells runnable for regression without assigning normal traffic. + await adminPost('/v1/admin/cell-state', { v: 1, cellId: targetCellId, enabled: false }) + } + process.stdout.write( + `${JSON.stringify({ + event: 'cell_deployed', + service: config.service, + sourceCellId, + targetCellId, + enabled: finalEnabled, + drainRevision: drainRevision.revision, + previousRevision: previous.revision, + candidateRevision: candidate.revision + })}\n` + ) +} + +export async function main(argv = process.argv.slice(2)) { + const config = parseArguments(argv) + const tag = cloudRunTrafficTag(config.service, 'candidate', config['release-id']) + const oldTag = cloudRunTrafficTag(config.service, 'previous', config['release-id']) + const drainTag = cloudRunTrafficTag(config.service, 'drain', config['release-id']) + if (config.role === 'director') await deployDirector(config, tag) + else await deployCell(config, tag, oldTag, drainTag) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs new file mode 100644 index 00000000000..6e56676098b --- /dev/null +++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs @@ -0,0 +1,814 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { + activeRevision, + cloudRunTrafficTag, + DIRECTOR_ADMISSION_ENVIRONMENT, + DIRECTOR_REGIONAL_PLACEMENT_ENV, + DIRECTOR_REGIONAL_PLACEMENT_SECRET, + DIRECTOR_REHOME_AUDIENCE_ENV, + DIRECTOR_REHOME_IDENTITY_ENV, + assertRegionalRehomeDisabled, + deployDirector, + directorDeploymentEnvironment, + directorCellSetAddition, + directorStartupProbeArguments, + directorTopologyChange, + environmentUpdateValue, + parseArguments, + revisionEnvironment, + revisionSecretEnvironment, + suppliedAdminIdentityToken, + taggedRevisionOrigin, + taggedTraffic, + trafficTags, + waitForEvacuationCapacity +} from './deploy-relay-blue-green.mjs' + +// Terraform declares these values; audited blue/green stamps the same values without targeting +// the drifted service, so this contract must fail before either side can silently diverge. +function terraformDirectorEnvironment(names) { + const read = (name) => + readFileSync(fileURLToPath(new URL(`../../infra/terraform/${name}`, import.meta.url)), 'utf8') + const relay = read('relay.tf') + const variables = read('variables.tf') + const production = read('environments/production.tfvars') + return Object.fromEntries( + names.map((name) => { + const block = new RegExp(`name\\s*=\\s*"${name}"\\s*\\n\\s*value\\s*=\\s*([^\\n]+)`).exec(relay) + assert.ok(block, `${name} is not set by relay.tf`) + const variable = /var\.([a-z_]+)/.exec(block[1]) + assert.ok(variable, `${name} is not sourced from a Terraform variable`) + // An environment override wins over the variable default, as Terraform resolves it. + const override = new RegExp(`^${variable[1]}\\s*=\\s*(\\S+)`, 'm').exec(production) + const fallback = new RegExp( + `variable\\s+"${variable[1]}"[\\s\\S]*?default\\s*=\\s*(\\S+)` + ).exec(variables) + assert.ok(override || fallback, `${variable[1]} has neither an override nor a default`) + return [name, String((override ?? fallback)[1]).replace(/"/g, '')] + }) + ) +} + +test('director admission environment matches what Terraform deploys', () => { + const names = Object.keys(DIRECTOR_ADMISSION_ENVIRONMENT) + assert.deepEqual(terraformDirectorEnvironment(names), { ...DIRECTOR_ADMISSION_ENVIRONMENT }) + + const relay = readFileSync( + fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), + 'utf8' + ) + assert.match( + relay, + /name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED"[\s\S]*?secret\s+= google_secret_manager_secret\.relay_regional_placement_enabled\.secret_id[\s\S]*?version = data\.external\.relay_serving_regional_placement_version\.result\.version/ + ) + assert.match(relay, /data "external" "relay_serving_regional_placement_version"/) + assert.match(relay, /read-relay-serving-regional-placement-version\.mjs/) + assert.doesNotMatch(relay, /template\[0\]\.containers\[0\]\.env/) +}) + +test('validates the final stamped-cell admission state', () => { + const required = [ + '--project', + 'project', + '--region', + 'region', + '--service', + 'service', + '--image', + 'image', + '--role', + 'cell', + '--release-id', + 'release', + '--director-origin', + 'https://relay.example.com', + '--admin-audience', + 'https://relay.example.com/v1/admin/drain' + ] + + assert.equal(parseArguments(required)['final-admission'], undefined) + assert.equal( + parseArguments([...required, '--final-admission', 'disabled'])['final-admission'], + 'disabled' + ) + assert.throws(() => parseArguments([...required, '--final-admission', 'sometimes'])) + assert.equal(parseArguments([...required, '--min-instances', '0'])['min-instances'], '0') + assert.throws(() => parseArguments([...required, '--min-instances', '-1'])) + assert.throws(() => + parseArguments([...required, '--capacity-service-account', 'relay@example.com']) + ) +}) + +test('validates optional director capacity configuration', () => { + const cells = [ + { + id: 'staging-gce-c3', + url: 'https://c3.relay-staging.onorca.dev', + capacityRequests: 4_000, + initiallyEnabled: false, + region: 'us-central1', + connectionHardCap: 600, + connectionUnobservedBound: 60 + } + ] + const config = { + project: 'onorca-cloud-staging', + 'capacity-service-account': + 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com', + 'director-cells-json': JSON.stringify(cells) + } + assert.deepEqual(directorDeploymentEnvironment(config), { + ...DIRECTOR_ADMISSION_ENVIRONMENT, + ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3', + ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT: + 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com', + ORCA_RELAY_CELLS_JSON: JSON.stringify([ + { + id: cells[0].id, + url: cells[0].url, + capacityRequests: cells[0].capacityRequests, + region: cells[0].region, + initiallyEnabled: cells[0].initiallyEnabled, + connectionHardCap: cells[0].connectionHardCap, + connectionUnobservedBound: cells[0].connectionUnobservedBound + } + ]) + }) + assert.throws( + () => + directorDeploymentEnvironment({ + ...config, + 'capacity-service-account': 'foreign@other-project.iam.gserviceaccount.com' + }), + /selected project/ + ) + assert.throws( + () => + directorDeploymentEnvironment({ + ...config, + 'director-cells-json': JSON.stringify([{ ...cells[0], unexpected: true }]) + }), + /invalid cell/ + ) + assert.match( + environmentUpdateValue(directorDeploymentEnvironment(config)), + /^\^~\^ORCA_RELAY_DATABASE_POOL_MAX=/ + ) + assert.equal(environmentUpdateValue({ FIRST: 'one', SECOND: 'two' }), 'FIRST=one,SECOND=two') + assert.deepEqual( + directorTopologyChange( + JSON.stringify([ + { + capacityRequests: 4_000, + connectionHardCap: 600, + connectionUnobservedBound: 60, + id: 'staging-gce-c3', + initiallyEnabled: false, + region: 'us-central1', + url: 'https://c3.relay-staging.onorca.dev' + } + ]), + JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]), + 'staging-gce-c3' + ), + { + changed: true, + value: directorDeploymentEnvironment({ + 'director-cells-json': JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]) + }).ORCA_RELAY_CELLS_JSON + } + ) + assert.throws( + () => + directorTopologyChange( + JSON.stringify(cells), + JSON.stringify([{ ...cells[0], url: 'https://wrong.relay-staging.onorca.dev' }]), + 'staging-gce-c3' + ), + /outside the reviewed capacity pair/ + ) +}) + +test('validates exact director runtime and regional rehome identities', () => { + const base = [ + '--project', + 'onorca-cloud', + '--region', + 'us-central1', + '--service', + 'orca-cloud-relay', + '--image', + `relay@sha256:${'a'.repeat(64)}`, + '--role', + 'director', + '--release-id', + 'rehome', + '--runtime-service-account', + 'relay-director@onorca-cloud.iam.gserviceaccount.com', + '--rehome-director-service-account', + 'relay-director@onorca-cloud.iam.gserviceaccount.com', + '--rehome-audience', + 'https://relay.onorca.dev/v1/admin/host-drain', + '--expected-rehome-generation', + '7', + '--rehome-control-origin', + 'https://relay.onorca.dev', + '--admin-audience', + 'https://relay.onorca.dev/v1/admin/drain' + ] + const config = parseArguments(base) + assert.deepEqual(directorDeploymentEnvironment(config), { + ...DIRECTOR_ADMISSION_ENVIRONMENT, + ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3', + ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`, + [DIRECTOR_REHOME_IDENTITY_ENV]: + 'relay-director@onorca-cloud.iam.gserviceaccount.com', + [DIRECTOR_REHOME_AUDIENCE_ENV]: + 'https://relay.onorca.dev/v1/admin/host-drain' + }) + const missingAudience = [...base] + missingAudience.splice(missingAudience.indexOf('--rehome-audience'), 2) + assert.throws(() => parseArguments(missingAudience), /configured together/) + const invalidOrigin = [...base] + invalidOrigin[invalidOrigin.indexOf('--rehome-control-origin') + 1] = + 'http://relay.onorca.dev' + assert.throws( + () => parseArguments(invalidOrigin), + /HTTPS origin/ + ) + const mutableImage = [...base] + mutableImage[mutableImage.indexOf('--image') + 1] = 'relay:latest' + assert.throws(() => parseArguments(mutableImage), /immutable digest/) +}) + +test('requires durable regional rehome control to be disabled at the exact generation', async () => { + const config = { + 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain', + 'expected-rehome-generation': '7' + } + const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN + process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' + try { + const control = await assertRegionalRehomeDisabled( + config, + 'https://candidate.example.test', + async (url, init) => { + assert.equal(url, 'https://candidate.example.test/v1/admin/regional-rehome-control') + assert.equal(init.headers.authorization, 'Bearer aaa.bbb.ccc') + return new Response(JSON.stringify({ + v: 1, + control: { generation: 7, enabled: false } + })) + } + ) + assert.equal(control.generation, 7) + await assert.rejects( + assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () => + new Response(JSON.stringify({ + v: 1, + control: { generation: 8, enabled: false } + })) + ), + /expected generation/ + ) + await assert.rejects( + assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () => + new Response(JSON.stringify({ + v: 1, + control: { generation: 7, enabled: true } + })) + ), + /durably disabled/ + ) + } finally { + if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN + else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment + } +}) + +test('rejects literal regional placement changes outside the runtime-setting step', () => { + const base = { + project: 'onorca-cloud', + region: 'us-central1', + service: 'orca-cloud-relay', + image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`, + role: 'director', + 'release-id': 'regional-kill-switch' + } + const args = Object.entries(base).flatMap(([key, value]) => [`--${key}`, value]) + + assert.doesNotThrow(() => parseArguments(args)) + assert.throws( + () => parseArguments([...args, '--regional-placement-enabled', 'false']), + /audited runtime-setting step/ + ) +}) + +test('appends disabled Asia cells without changing the existing director topology', () => { + const current = [ + { + id: 'production-gce-c26', + url: 'https://c26.relay.onorca.dev', + capacityRequests: 4_000, + initiallyEnabled: true, + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + } + ] + const asia = { + id: 'production-gce-c27', + url: 'https://c27.relay.onorca.dev', + region: 'asia-east2', + capacityRequests: 6_000, + initiallyEnabled: false, + connectionHardCap: 3_000, + connectionUnobservedBound: 60 + } + + assert.deepEqual( + directorCellSetAddition( + JSON.stringify(current), + JSON.stringify([asia, { ...current[0], region: 'us-central1' }]) + ), + { + changed: true, + value: directorDeploymentEnvironment({ + 'director-cells-json': JSON.stringify([asia, { ...current[0], region: 'us-central1' }]) + }).ORCA_RELAY_CELLS_JSON + } + ) + const exact = JSON.stringify([{ ...current[0], region: 'us-central1' }, asia]) + assert.deepEqual(directorCellSetAddition(exact, exact), { + changed: false, + value: directorDeploymentEnvironment({ 'director-cells-json': exact }) + .ORCA_RELAY_CELLS_JSON + }) + assert.throws( + () => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1', capacityRequests: 6_000 }, asia])), + /changes an existing cell/ + ) + assert.throws( + () => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1' }, { ...asia, initiallyEnabled: true }])), + /must start disabled/ + ) +}) + +test('pins a director startup probe above the bounded reconciliation window', () => { + assert.deepEqual(directorStartupProbeArguments('director'), [ + '--startup-probe', + 'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1' + ]) + assert.deepEqual(directorStartupProbeArguments('cell'), []) +}) + +test('bounds traffic tags by the Cloud Run service-plus-tag contract', () => { + const service = 'orca-cloud-relay-staging-c1' + const candidate = cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a') + assert.match(candidate, /^candidate-[a-f0-9]{9}$/) + assert.equal(service.length + candidate.length, 46) + assert.equal(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a')) + assert.notEqual(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-2-19cc312a')) + assert.throws(() => cloudRunTrafficTag(`${service}-too-long`, 'candidate', 'release')) +}) + +test('derives and validates a Cloud Run tagged revision origin', () => { + assert.equal( + taggedRevisionOrigin( + 'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app', + 'candidate-123' + ), + 'https://candidate-123---orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app' + ) + assert.throws(() => taggedRevisionOrigin('https://relay-staging.onorca.dev', 'candidate-123')) + assert.throws(() => + taggedRevisionOrigin( + 'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app', + '123-invalid' + ) + ) +}) + +test('requires exactly one active revision and reads queried tag metadata', () => { + const service = { + status: { + traffic: [ + { percent: 100, revisionName: 'relay-00001-old' }, + { + percent: 0, + revisionName: 'relay-00002-new', + tag: 'candidate-123', + url: 'https://candidate-123---relay-hash-uc.a.run.app' + } + ] + } + } + assert.equal(activeRevision(service), 'relay-00001-old') + assert.deepEqual(taggedTraffic(service, 'candidate-123'), { + origin: 'https://candidate-123---relay-hash-uc.a.run.app', + revision: 'relay-00002-new' + }) + assert.throws(() => + activeRevision({ status: { traffic: [{ percent: 50 }, { percent: 50 }] } }) + ) + assert.deepEqual(trafficTags(service), ['candidate-123']) +}) + +function directorHarness({ + deployFailure, + deleteFailure, + cleanupReportsFailure = false, + cleanupFailsBeforeRemoval = false, + servingMinimum = 1, + servingMaximum = 5, + // Reproduces gcloud dropping minScale from a newly created revision. + dropRequestedMinimum = false, + servingServiceAccount, + servingImageDigest = `sha256:${'f'.repeat(64)}` +} = {}) { + const state = { + activeRevision: 'relay-00001-old', + tags: new Map([['candidate-old', 'relay-00000-stale']]), + revisions: new Map([ + ['relay-00000-stale', { env: { ORCA_RELAY_ROLE: 'director' }, minimum: 1, maximum: 5 }], + [ + 'relay-00001-old', + { + env: { + ORCA_RELAY_ROLE: 'director' + }, + secrets: { + [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { + secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, + version: '1' + } + }, + minimum: servingMinimum, + maximum: servingMaximum, + serviceAccount: servingServiceAccount, + image: `relay@${servingImageDigest}` + } + ] + ]), + nextRevision: 2 + } + const removed = [] + const healthProtocols = [] + let pendingCleanupFailure = cleanupFailsBeforeRemoval + const operations = { + describeService: () => ({ + status: { + traffic: [ + { percent: 100, revisionName: state.activeRevision }, + ...[...state.tags].map(([tag, revisionName]) => ({ + tag, + revisionName, + url: `https://${tag}---relay-hash-uc.a.run.app` + })) + ] + } + }), + describeRevision: (_config, revision) => { + const value = state.revisions.get(revision) + return { + metadata: { + annotations: { + 'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0), + 'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5) + } + }, + spec: { + serviceAccountName: value?.serviceAccount, + containers: [ + { + image: value?.image, + env: [ + ...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })), + ...Object.entries(value?.secrets ?? {}).map(([name, secretKeyRef]) => ({ + name, + valueSource: { secretKeyRef } + })) + ] + } + ] + } + } + }, + deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => { + const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new` + state.revisions.set(revision, { + env: { ORCA_RELAY_ROLE: 'director', ...env }, + secrets: { + [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { + secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, + version: regionalVersion + } + }, + minimum: dropRequestedMinimum ? 0 : Number(minimum ?? 1), + maximum, + serviceAccount: + config['runtime-service-account'] ?? + state.revisions.get(state.activeRevision)?.serviceAccount, + image: image ?? state.revisions.get(state.activeRevision)?.image + }) + state.tags.set(tag, revision) + if (deployFailure) throw deployFailure + }, + listRevisions: () => + [...state.revisions].map(([name]) => ({ metadata: { name } })), + deleteRevision: (_config, revision) => { + if (deleteFailure) throw deleteFailure + state.revisions.delete(revision) + }, + updateTraffic: (_config, args) => { + const remove = args.find((argument) => argument.startsWith('--remove-tags=')) + if (remove) { + const tags = remove.slice('--remove-tags='.length).split(',') + removed.push(tags) + if (pendingCleanupFailure && tags.includes('candidate-new')) { + pendingCleanupFailure = false + throw new Error('failed to remove promoted candidate tag') + } + for (const tag of tags) state.tags.delete(tag) + if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision') + return + } + const promote = args.find((argument) => argument.startsWith('--to-tags=')) + assert.ok(promote) + const tag = promote.slice('--to-tags='.length).split('=')[0] + state.activeRevision = state.tags.get(tag) + }, + waitForHealth: async (_origin, connectionCapacityProtocol) => { + healthProtocols.push(connectionCapacityProtocol) + } + } + return { state, removed, healthProtocols, operations } +} + +test('director deploy removes stale and promoted Cloud Run tags', async () => { + const harness = directorHarness() + const config = { + project: 'onorca-cloud-staging', + 'capacity-service-account': + 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com' + } + await deployDirector(config, 'candidate-new', harness.operations) + assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']]) + assert.equal(harness.state.activeRevision, 'relay-00003-new') + assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) + assert.deepEqual([...harness.state.revisions.keys()], [ + 'relay-00000-stale', + 'relay-00001-old', + 'relay-00002-new', + 'relay-00003-new' + ]) + for (const revision of ['relay-00002-new', 'relay-00003-new']) { + assert.deepEqual( + Object.fromEntries( + Object.entries(harness.state.revisions.get(revision).env).filter(([key]) => + key in DIRECTOR_ADMISSION_ENVIRONMENT + ) + ), + DIRECTOR_ADMISSION_ENVIRONMENT + ) + assert.equal( + harness.state.revisions.get(revision).env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT, + config['capacity-service-account'] + ) + } + assert.deepEqual(harness.healthProtocols, [undefined, undefined]) +}) + +test('director deploy stamps the durable regional placement secret reference', async () => { + const harness = directorHarness() + await deployDirector({}, 'candidate-new', harness.operations) + for (const revision of ['relay-00002-new', 'relay-00003-new']) { + assert.deepEqual( + harness.state.revisions.get(revision).secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV], + { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: '1' } + ) + } +}) + +test('bootstraps both rollback and candidate onto the distinct director identity', async () => { + const predecessor = 'relay-runtime@onorca-cloud.iam.gserviceaccount.com' + const director = 'relay-director@onorca-cloud.iam.gserviceaccount.com' + const harness = directorHarness({ servingServiceAccount: predecessor }) + const inspected = [] + harness.operations.assertRegionalRehomeDisabled = async (_config, origin) => { + inspected.push(origin) + } + await deployDirector({ + project: 'onorca-cloud', + 'runtime-service-account': director, + 'predecessor-runtime-service-account': predecessor, + 'predecessor-image-digest': `sha256:${'f'.repeat(64)}`, + 'bootstrap-runtime-identity': 'true', + 'expected-rehome-generation': '0', + 'rehome-control-origin': 'https://relay.onorca.dev' + }, 'candidate-new', harness.operations) + assert.equal( + harness.state.revisions.get(harness.state.activeRevision).serviceAccount, + director + ) + assert.equal( + harness.state.revisions.get(harness.state.tags.get('selector-rollback')).serviceAccount, + director + ) + assert.deepEqual(inspected, [ + 'https://selector-rollback---relay-hash-uc.a.run.app', + 'https://candidate-new---relay-hash-uc.a.run.app' + ]) +}) + +test('steady-state director deploy rejects the predecessor identity', async () => { + const harness = directorHarness({ + servingServiceAccount: 'relay-runtime@onorca-cloud.iam.gserviceaccount.com' + }) + await assert.rejects(deployDirector({ + 'runtime-service-account': 'relay-director@onorca-cloud.iam.gserviceaccount.com' + }, 'candidate-new', harness.operations), /unexpected runtime service account/) +}) + +test('director deploy prunes old revisions when requested', async () => { + const harness = directorHarness() + await deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations) + assert.deepEqual([...harness.state.revisions.keys()], [ + 'relay-00002-new', + 'relay-00003-new' + ]) + assert.deepEqual(harness.healthProtocols, [2, 2]) +}) + +test('a prune failure preserves the active and rollback traffic pair', async () => { + const harness = directorHarness({ deleteFailure: new Error('injected delete failure') }) + await assert.rejects( + deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations), + /injected delete failure/ + ) + assert.equal(harness.state.activeRevision, 'relay-00003-new') + assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) +}) + +test('director deploy removes a candidate tag left by a failed update', async () => { + const failure = new Error('candidate failed to become ready') + const harness = directorHarness({ deployFailure: failure }) + await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure) + assert.deepEqual(harness.removed, [['candidate-old'], ['selector-rollback']]) + assert.equal(harness.state.tags.size, 0) +}) + +test('director candidate inherits the serving warm-instance floor', async () => { + const harness = directorHarness({ servingMinimum: 5 }) + await deployDirector({}, 'candidate-new', harness.operations) + const serving = harness.state.revisions.get(harness.state.activeRevision) + assert.equal(serving.minimum, 5) + // The standby rollback revision must stay cold. + const rollback = harness.state.revisions.get(harness.state.tags.get('selector-rollback')) + assert.equal(rollback.minimum, 0) +}) + +test('director deploy rejects a serving maximum above the checked budget', async () => { + const harness = directorHarness({ servingMaximum: 6 }) + await assert.rejects( + deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations), + /holds 6 maximum instances, expected 5/ + ) +}) + +test('an explicit --min-instances still overrides the serving floor', async () => { + const harness = directorHarness({ servingMinimum: 5 }) + await deployDirector({ 'min-instances': '0' }, 'candidate-new', harness.operations) + assert.equal(harness.state.revisions.get(harness.state.activeRevision).minimum, 0) +}) + +test('director deploy refuses to move traffic onto a candidate that lost the floor', async () => { + const harness = directorHarness({ servingMinimum: 5, dropRequestedMinimum: true }) + await assert.rejects( + deployDirector({}, 'candidate-new', harness.operations), + /candidate holds 0 minimum instances, expected 5/ + ) + // Traffic never moved, so the original revision still serves. + assert.equal(harness.state.activeRevision, 'relay-00001-old') +}) + +test('director deploy rejects unrelated revision-shape drift', async () => { + const harness = directorHarness() + const describeRevision = harness.operations.describeRevision + harness.operations.describeRevision = (config, revision) => { + const described = describeRevision(config, revision) + described.spec.containerConcurrency = revision === 'relay-00001-old' ? 80 : 1_000 + return described + } + await assert.rejects( + deployDirector({}, 'candidate-new', harness.operations), + /unrelated revision shape/ + ) + assert.equal(harness.state.activeRevision, 'relay-00001-old') +}) + +test('director cleanup verifies success when gcloud reports a stale revision failure', async () => { + const harness = directorHarness({ cleanupReportsFailure: true }) + await deployDirector({}, 'candidate-new', harness.operations) + assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']]) + assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) +}) + +test('director deploy restores rollback traffic after promoted-tag cleanup fails', async () => { + const harness = directorHarness({ cleanupFailsBeforeRemoval: true }) + await assert.rejects( + deployDirector({}, 'candidate-new', harness.operations), + /failed to remove promoted candidate tag/ + ) + assert.equal( + harness.state.activeRevision, + harness.state.tags.get('selector-rollback') + ) + assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) +}) + +test('reads only literal revision environment values', () => { + assert.deepEqual( + revisionEnvironment({ + spec: { + containers: [ + { + env: [ + { name: 'ORCA_RELAY_CELL_ID', value: 'staging-c1' }, + { name: 'ORCA_RELAY_CELL_CAPACITY', value: '900' }, + { name: 'DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database' } } } + ] + } + ] + } + }), + { ORCA_RELAY_CELL_ID: 'staging-c1', ORCA_RELAY_CELL_CAPACITY: '900' } + ) +}) + +test('reads only Secret Manager revision environment references', () => { + assert.deepEqual( + revisionSecretEnvironment({ + spec: { + containers: [{ env: [ + { name: 'LITERAL', value: 'true' }, + { + name: DIRECTOR_REGIONAL_PLACEMENT_ENV, + valueSource: { secretKeyRef: { + secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, + version: 'latest' + } } + }, + { + name: 'GCP_SECRET_SHAPE', + valueFrom: { secretKeyRef: { name: 'gcp-secret', key: '2' } } + } + ] }] + } + }), + { + [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { + secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, + version: 'latest' + }, + GCP_SECRET_SHAPE: { + secret: 'gcp-secret', + version: '2' + } + } + ) +}) + +test('accepts only a bounded JWT-shaped supplied admin identity token', () => { + assert.equal(suppliedAdminIdentityToken({}), null) + assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc') + assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' })) + assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' })) + assert.throws(() => + suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` }) + ) +}) + +test('waits for authenticated target readiness without hiding other capacity errors', async () => { + let attempts = 0 + const capacity = await waitForEvacuationCapacity( + async () => { + attempts += 1 + if (attempts < 3) throw new Error('/v1/admin/evacuation-capacity failed: target_cell_unavailable') + return { requiredTargetUnits: 2, availableTargetUnits: 4_000 } + }, + 'source', + 'target', + { pollIntervalMs: 1, timeoutMs: 100 } + ) + assert.equal(attempts, 3) + assert.equal(capacity.availableTargetUnits, 4_000) + await assert.rejects( + waitForEvacuationCapacity(async () => { + throw new Error('/v1/admin/evacuation-capacity failed: forbidden') + }, 'source', 'target'), + /forbidden/ + ) +}) diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.mjs new file mode 100644 index 00000000000..6a0928041c9 --- /dev/null +++ b/cloud/dev/scripts/deploy-relay-gce-candidate.mjs @@ -0,0 +1,871 @@ +import { readFileSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { pathToFileURL } from 'node:url' +import { + inspectAdmissionSelector, + selectorCellState, + transitionAdmissionSelector +} from './relay-admission-selector.mjs' + +const DEFAULT_POLL_INTERVAL_MS = 5_000 +const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000 +const ADMIN_RETRY_ATTEMPTS = 3 +const ADMIN_RETRY_BASE_MS = 250 +const CONNECTION_CONTROL_REBIND_RESERVE = 100 +const SUPPORTED_CONNECTION_HARD_CAPS = new Set([600, 1_000, 3_000]) +const RETRYABLE_ADMIN_PATHS = new Set([ + '/v1/admin/runtime-status', + '/v1/admin/cell-status', + '/v1/admin/evacuation-capacity', + '/v1/admin/evacuation-status' +]) + +function canonicalOrigin(value, name) { + const url = new URL(value) + if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') { + throw new Error(`${name} must be a canonical HTTPS origin`) + } + return value +} + +function adminAudience(value) { + const url = new URL(value) + if ( + url.protocol !== 'https:' || + url.pathname !== '/v1/admin/drain' || + url.search || + url.hash || + url.toString() !== value + ) { + throw new Error('--admin-audience must be the canonical HTTPS director drain URL') + } + return value +} + +function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) { + throw new Error(`${name} must be a positive integer`) + } + return parsed +} + +function nonnegativeInteger(value, name) { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a nonnegative integer`) + } + return parsed +} + +export function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) + values[key.slice(2)] = value + } + for (const key of [ + 'project', + 'director-origin', + 'admin-audience', + 'topology-file', + 'source-cell-id', + 'target-cell-id', + 'runtime-service-account', + 'mode' + ]) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if ( + ![ + 'audit', + 'preflight', + 'recover-forward', + 'continue-evacuation', + 'disable-cell', + 'execute', + 'reset-empty-candidate', + 'enable-empty-cell' + ].includes(values.mode) + ) { + throw new Error( + '--mode must be audit, preflight, recover-forward, continue-evacuation, disable-cell, execute, reset-empty-candidate, or enable-empty-cell' + ) + } + return { + project: values.project, + directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'), + adminAudience: adminAudience(values['admin-audience']), + topologyFile: values['topology-file'], + sourceCellId: values['source-cell-id'], + targetCellId: values['target-cell-id'], + runtimeServiceAccount: values['runtime-service-account'], + mode: values.mode, + batchSize: positiveInteger(values['batch-size'] ?? 100, '--batch-size', 100), + drainGraceMs: positiveInteger( + values['drain-grace-ms'] ?? 120_000, + '--drain-grace-ms', + 60 * 60 * 1_000 + ), + pollIntervalMs: positiveInteger( + values['poll-interval-ms'] ?? DEFAULT_POLL_INTERVAL_MS, + '--poll-interval-ms', + 60_000 + ), + timeoutMs: positiveInteger( + values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS, + '--timeout-ms', + 60 * 60 * 1_000 + ) + } +} + +export function deployment(value, cellId) { + if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`) + const expected = { + cellId, + origin: canonicalOrigin(value.origin, `${cellId} origin`), + region: String(value.region ?? 'us-central1'), + zone: String(value.zone ?? ''), + migName: String(value.mig_name ?? ''), + instanceGroup: String(value.instance_group ?? ''), + backendName: String(value.backend_name ?? ''), + backendId: String(value.backend_id ?? ''), + urlMapName: String(value.url_map_name ?? ''), + generationIdentity: String(value.generation_identity ?? ''), + image: String(value.image ?? ''), + imageDigest: String(value.image ?? '').split('@')[1] ?? '', + capacityRequests: positiveInteger(value.capacity_requests, `${cellId} capacity`), + databasePoolMax: positiveInteger( + value.database_pool_max ?? 10, + `${cellId} database pool maximum`, + 100 + ), + connectionHardCap: + value.connection_hard_cap === null || value.connection_hard_cap === undefined + ? undefined + : positiveInteger(value.connection_hard_cap, `${cellId} connection hard cap`), + connectionUnobservedBound: + value.connection_unobserved_bound === null || + value.connection_unobserved_bound === undefined + ? undefined + : nonnegativeInteger( + value.connection_unobserved_bound, + `${cellId} unobserved connection bound` + ), + initiallyEnabled: value.initially_enabled, + fenced: value.fenced, + desiredTargetSize: value.desired_target_size + } + if (!/^[a-z0-9-]+$/.test(expected.zone)) throw new Error(`${cellId} has an invalid zone`) + if (!['us-central1', 'asia-east2'].includes(expected.region) || !expected.zone.startsWith(`${expected.region}-`)) { + throw new Error(`${cellId} has an invalid region`) + } + for (const [name, resource] of [ + ['MIG', expected.migName], + ['instance group', expected.instanceGroup], + ['backend', expected.backendName], + ['backend ID', expected.backendId] + ]) { + if (!resource) throw new Error(`${cellId} has no ${name}`) + } + if (!/^[a-z0-9.-]+\/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$/.test(expected.image)) { + throw new Error(`${cellId} image is not digest-pinned`) + } + if (typeof expected.initiallyEnabled !== 'boolean') { + throw new Error(`${cellId} has no initial admission state`) + } + if ( + (expected.connectionHardCap === undefined) !== + (expected.connectionUnobservedBound === undefined) || + (expected.connectionHardCap !== undefined && + (!SUPPORTED_CONNECTION_HARD_CAPS.has(expected.connectionHardCap) || + expected.connectionUnobservedBound >= + expected.connectionHardCap - CONNECTION_CONTROL_REBIND_RESERVE)) + ) { + throw new Error(`${cellId} has invalid connection capacity`) + } + return expected +} + +export function assertDeploymentConnectionCapacity(expected, runtime, director) { + if (expected.connectionHardCap === undefined) { + if (runtime !== null || director !== null) { + throw new Error(`${expected.cellId} connection capacity differs from Terraform`) + } + return + } + const hardCap = expected.connectionHardCap + const unobservedBound = expected.connectionUnobservedBound + const ordinaryConnectionLimit = hardCap - CONNECTION_CONTROL_REBIND_RESERVE + const normalAdmissionPause = ordinaryConnectionLimit - unobservedBound + const matches = (capacity) => + capacity?.hardCap === hardCap && + capacity.controlRebindReserve === CONNECTION_CONTROL_REBIND_RESERVE && + capacity.ordinaryConnectionLimit === ordinaryConnectionLimit && + capacity.unobservedBound === unobservedBound && + capacity.normalAdmissionPause === normalAdmissionPause + if (!matches(runtime) || !matches(director) || director.heartbeatFresh !== true) { + throw new Error(`${expected.cellId} connection capacity differs from Terraform`) + } +} + +export function selectDeployments(topology, sourceCellId, targetCellId) { + if (sourceCellId === targetCellId) throw new Error('source and target cell IDs must differ') + const source = deployment(topology[sourceCellId], sourceCellId) + const target = deployment(topology[targetCellId], targetCellId) + for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) { + if (source[key] === target[key]) throw new Error(`source and target ${key} overlap`) + } + if (target.initiallyEnabled) throw new Error('candidate must be declared initially disabled') + return { source, target } +} + +export function validateMig(mig, instances, expected) { + if (Number(mig.targetSize) !== 1) throw new Error(`${expected.cellId} MIG is not fixed-one`) + const policy = mig.updatePolicy ?? {} + if ( + policy.replacementMethod !== 'RECREATE' || + Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 || + Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1 + ) { + throw new Error(`${expected.cellId} MIG replacement policy is unsafe`) + } + const serving = instances.filter( + (entry) => entry.instanceStatus === 'RUNNING' && entry.currentAction === 'NONE' + ) + if (instances.length !== 1 || serving.length !== 1) { + throw new Error(`${expected.cellId} MIG must have one running endpoint`) + } + return serving[0].instance.split('/').at(-1) +} + +export function validateInstance(instance, expected, runtimeServiceAccount) { + const publicConfigs = (instance.networkInterfaces ?? []).flatMap( + (network) => network.accessConfigs ?? [] + ) + if (publicConfigs.length !== 0) throw new Error(`${expected.cellId} instance has a public IP`) + const serviceAccounts = (instance.serviceAccounts ?? []).map((entry) => entry.email) + if (serviceAccounts.length !== 1 || serviceAccounts[0] !== runtimeServiceAccount) { + throw new Error(`${expected.cellId} runtime service account mismatch`) + } +} + +export function validateBackend(backend, expected) { + if ( + backend.protocol !== 'HTTP' || + Number(backend.timeoutSec) !== 86_400 || + (backend.backends ?? []).length !== 1 || + backend.backends[0].group !== expected.instanceGroup + ) { + throw new Error(`${expected.cellId} backend topology mismatch`) + } +} + +export function defaultCommandJson(args) { + const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) + if (result.status !== 0) { + throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`) + } + return JSON.parse(result.stdout) +} + +export function suppliedAdminIdentityToken(environment = process.env) { + const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN + if (token === undefined) return null + return validatedIdentityToken(token, 'admin') +} + +export function suppliedFenceMutationIdentityToken(environment = process.env) { + const token = environment.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN + if (token === undefined) return null + return validatedIdentityToken(token, 'fence mutation') +} + +function validatedIdentityToken(token, label) { + // WIF supplies a masked Google ID token because external-account gcloud cannot mint one directly. + if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) { + throw new Error(`invalid supplied ${label} identity token`) + } + return token +} + +export function defaultIdentityToken(audience) { + const supplied = suppliedAdminIdentityToken() + if (supplied !== null) return supplied + const result = spawnSync( + 'gcloud', + ['auth', 'print-identity-token', `--audiences=${audience}`], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] } + ) + if (result.status !== 0) throw new Error('gcloud identity-token command failed') + return result.stdout.trim() +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({ error: `http_${response.status}` })) + if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`) + return body +} + +export function createAdminPost(config, deps, token) { + return async (origin, path, body) => { + const requestToken = typeof token === 'function' ? token(path) : token + for (let attempt = 1; attempt <= ADMIN_RETRY_ATTEMPTS; attempt++) { + let response + try { + response = await deps.fetch(`${origin}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${requestToken}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }) + } catch (error) { + if (!RETRYABLE_ADMIN_PATHS.has(path) || attempt === ADMIN_RETRY_ATTEMPTS) throw error + deps.emit({ event: 'candidate_admin_retry', path, attempt, reason: 'transport' }) + await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1)) + continue + } + if ( + RETRYABLE_ADMIN_PATHS.has(path) && + ([502, 503, 504].includes(response.status) || + (path === '/v1/admin/evacuation-status' && response.status === 500)) && + attempt < ADMIN_RETRY_ATTEMPTS + ) { + // These endpoints are read-only or transactionally idempotent, so a + // lost response may be retried without widening deployment authority. + deps.emit({ + event: 'candidate_admin_retry', + path, + attempt, + reason: `http_${response.status}` + }) + await response.arrayBuffer().catch(() => undefined) + await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1)) + continue + } + return await responseJson(response, path) + } + throw new Error(`${path} retry attempts exhausted`) + } +} + +async function checkHttp(deps, origin, path) { + const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) }) + const body = await response.json().catch(() => ({})) + if (!response.ok || body.ok !== true) throw new Error(`${origin}${path} is unavailable`) +} + +export async function inspectCell(config, deps, adminPost, expected) { + const common = ['--project', config.project, '--zone', expected.zone, '--format=json'] + const mig = deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'describe', + expected.migName, + ...common + ]) + const instances = deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'list-instances', + expected.migName, + ...common + ]) + const instanceName = validateMig(mig, instances, expected) + const instance = deps.commandJson([ + 'compute', + 'instances', + 'describe', + instanceName, + ...common + ]) + validateInstance(instance, expected, config.runtimeServiceAccount) + const backend = deps.commandJson([ + 'compute', + 'backend-services', + 'describe', + expected.backendName, + '--global', + '--project', + config.project, + '--format=json' + ]) + validateBackend(backend, expected) + await checkHttp(deps, expected.origin, '/health') + await checkHttp(deps, expected.origin, '/ready') + const runtime = await adminPost(expected.origin, '/v1/admin/runtime-status', { v: 1 }) + if ( + runtime.role !== 'cell' || + runtime.cellId !== expected.cellId || + runtime.cellUrl !== expected.origin || + (runtime.region ?? 'us-central1') !== expected.region || + runtime.imageDigest !== expected.imageDigest + ) { + throw new Error(`${expected.cellId} served runtime does not match Terraform topology`) + } + const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: expected.cellId + }) + if ( + status.status?.cellUrl !== expected.origin || + (status.status?.region ?? 'us-central1') !== expected.region || + status.status?.runtime?.cellUrl !== expected.origin || + status.status?.runtime?.ready !== true || + status.status?.runtime?.heartbeatFresh !== true + ) { + throw new Error(`${expected.cellId} has no fresh ready authenticated heartbeat`) + } + assertDeploymentConnectionCapacity( + expected, + runtime.connectionCapacity ?? null, + status.status.connectionCapacity ?? null + ) + return { + ...status.status, + draining: runtime.draining === true, + process: runtime.runtime ?? null, + runtimeConnectionCapacity: runtime.connectionCapacity ?? null + } +} + +async function waitForMigration(config, deps, adminPost, completeReady) { + const deadline = deps.now() + config.timeoutMs + while (deps.now() < deadline) { + const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { + v: 1, + sourceCellId: config.sourceCellId, + targetCellId: config.targetCellId, + completeReady + }) + deps.emit({ event: completeReady ? 'migration_completion' : 'migration_registration', ...status }) + if (completeReady ? status.inProgress === 0 : status.inProgress === status.targetRegistered) { + return status + } + if ( + completeReady && + status.targetRegistered === status.inProgress && + status.registeredSourceActive === 0 && + status.registeredCompletable === 0 && + status.registeredTargetInactive === status.inProgress + ) { + // CI waiting cannot revive an offline desktop; keep its proven migration + // pending until that target control reconnects. + return status + } + await deps.wait(config.pollIntervalMs) + } + throw new Error('timed out waiting for candidate migration') +} + +export async function setCellState(config, adminPost, cellId, enabled) { + const post = async (path, body) => await adminPost(config.directorOrigin, path, body) + const inspected = await inspectAdmissionSelector(post) + if (inspected.selector.generation > 0) { + await transitionAdmissionSelector(post, { + [cellId]: enabled ? 'general' : 'existing-only' + }) + return + } + await post('/v1/admin/cell-state', { v: 1, cellId, enabled }) +} + +function assertNoDurableActivity(status, operation) { + const activity = [ + status.assignments, + status.activityLeases, + status.reservedRequests, + status.outgoingMigrations, + status.incomingMigrations + ] + if (activity.some((value) => Number(value) !== 0)) { + throw new Error(`${operation} requires zero durable activity`) + } +} + +async function recoverCandidateFailure( + config, + deps, + adminPost, + source, + target, + allowEmptyAdmissionRollback, + selectorActive +) { + const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { + v: 1, + sourceCellId: source.cellId, + targetCellId: target.cellId, + completeReady: false + }).catch(() => null) + if (!selectorActive && allowEmptyAdmissionRollback && status?.inProgress === 0) { + await setCellState(config, adminPost, source.cellId, true).catch(() => undefined) + await setCellState(config, adminPost, target.cellId, false).catch(() => undefined) + return + } + if (!selectorActive && status && status.inProgress > 0 && status.targetRegistered === 0) { + await setCellState(config, adminPost, source.cellId, true).catch(() => undefined) + deps.emit({ + event: 'candidate_rollback_waiting_for_lease_expiry', + sourceCellId: source.cellId, + targetCellId: target.cellId, + inProgress: status.inProgress + }) + return + } + deps.emit({ + event: 'candidate_forward_recovery_required', + sourceCellId: source.cellId, + targetCellId: target.cellId, + targetRegistered: status?.targetRegistered ?? null + }) +} + +export async function drainSource( + config, + deps, + token, + source, + graceMs = config.drainGraceMs, + traceValue +) { + const response = await deps.fetch(`${source.origin}/v1/admin/drain`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + ...(traceValue ? { 'x-orca-drain-trace': traceValue } : {}) + }, + body: JSON.stringify({ v: 1, graceMs }), + signal: AbortSignal.timeout(30_000) + }) + await responseJson(response, 'source drain') + return { + backendStatus: response.status, + backendInstance: response.headers.get('x-orca-backend-instance') ?? undefined + } +} + +async function verifyCandidateCompletion( + config, + adminPost, + source, + target, + event, + eventName = 'candidate_complete' +) { + const finalSource = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: source.cellId + }) + const finalTarget = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: target.cellId + }) + if ( + finalSource.status.activityLeases !== 0 || + finalSource.status.reservedRequests !== 0 || + finalSource.status.outgoingMigrations !== 0 || + finalSource.status.runtime?.observedRequests !== 0 || + finalTarget.status.incomingMigrations !== 0 || + finalTarget.status.reservedRequests !== finalTarget.status.activityRequestUnits + ) { + throw new Error('aggregate post-migration counts are not reconciled') + } + event({ + event: eventName, + sourceCellId: source.cellId, + targetCellId: target.cellId, + dormantSourceAssignments: finalSource.status.assignments, + targetAssignments: finalTarget.status.assignments, + targetActivityLeases: finalTarget.status.activityLeases, + targetReservedRequests: finalTarget.status.reservedRequests + }) +} + +export async function runCandidateDeployment(config, overrides = {}) { + const deps = { + commandJson: overrides.commandJson ?? defaultCommandJson, + identityToken: overrides.identityToken ?? defaultIdentityToken, + fetch: overrides.fetch ?? fetch, + emit: + overrides.emit ?? + ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)), + now: overrides.now ?? Date.now, + wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + random: overrides.random ?? Math.random + } + const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8')) + const { source, target } = selectDeployments( + topology, + config.sourceCellId, + config.targetCellId + ) + const token = deps.identityToken(config.adminAudience) + const adminPost = createAdminPost(config, deps, token) + const selectorPost = async (path, body) => + await adminPost(config.directorOrigin, path, body) + const selectorInspection = await inspectAdmissionSelector(selectorPost) + const selectorActive = selectorInspection.selector.generation > 0 + const sourceStatus = await inspectCell(config, deps, adminPost, source) + const targetStatus = await inspectCell(config, deps, adminPost, target) + const sourceAdmission = selectorActive + ? selectorCellState(selectorInspection.selector, source.cellId) + : sourceStatus.enabled + ? 'general' + : 'existing-only' + const targetAdmission = selectorActive + ? selectorCellState(selectorInspection.selector, target.cellId) + : targetStatus.enabled + ? 'general' + : 'existing-only' + if (config.mode === 'audit') { + const migration = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { + v: 1, + sourceCellId: source.cellId, + targetCellId: target.cellId, + completeReady: false + }) + // Forward recovery needs durable aggregate evidence without exposing assignment identities. + deps.emit({ + event: 'candidate_audit', + source: aggregateCellStatus(sourceStatus), + target: aggregateCellStatus(targetStatus), + migration + }) + return + } + if (config.mode === 'recover-forward') { + if ( + selectorActive + ? sourceAdmission !== 'existing-only' || targetAdmission !== 'migration-only' + : sourceStatus.enabled || !targetStatus.enabled + ) { + throw new Error( + selectorActive + ? 'forward recovery requires existing-only source and migration-only target' + : 'forward recovery requires disabled source and enabled target' + ) + } + await drainSource(config, deps, token, source) + await waitForMigration(config, deps, adminPost, false) + const completion = await waitForMigration(config, deps, adminPost, true) + if (completion.inProgress > 0) { + deps.emit({ + event: 'candidate_forward_pending', + sourceCellId: source.cellId, + targetCellId: target.cellId, + inProgress: completion.inProgress, + registeredSourceActive: completion.registeredSourceActive, + registeredCompletable: completion.registeredCompletable, + registeredTargetInactive: completion.registeredTargetInactive + }) + throw new Error('forward recovery remains pending for inactive target controls') + } + await verifyCandidateCompletion( + config, + adminPost, + source, + target, + deps.emit, + 'candidate_forward_recovered' + ) + return + } + if (config.mode === 'disable-cell') { + if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false) + // Disabling new admission preserves origin-owned sessions and durable recovery work. + deps.emit({ + event: 'cell_admission_disabled', + targetCellId: target.cellId, + changed: targetStatus.enabled, + assignments: targetStatus.assignments, + activityLeases: targetStatus.activityLeases, + reservedRequests: targetStatus.reservedRequests, + outgoingMigrations: targetStatus.outgoingMigrations, + incomingMigrations: targetStatus.incomingMigrations + }) + return + } + // Repair is safe only before a candidate owns assignments or origin-scoped work. + if (config.mode === 'reset-empty-candidate') { + assertNoDurableActivity(targetStatus, 'candidate admission reset') + if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false) + deps.emit({ + event: 'candidate_admission_reset', + targetCellId: target.cellId, + changed: targetStatus.enabled + }) + return + } + if (config.mode === 'enable-empty-cell') { + assertNoDurableActivity(targetStatus, 'cell admission enable') + if (selectorActive ? targetAdmission === 'general' : targetStatus.enabled) { + deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: false }) + return + } + } + const continuingEvacuation = config.mode === 'continue-evacuation' + if (continuingEvacuation) { + if ( + selectorActive + ? targetAdmission !== 'migration-only' + : !targetStatus.enabled + ) { + throw new Error( + selectorActive + ? 'continued evacuation requires migration-only target' + : 'continued evacuation requires enabled target' + ) + } + } else if (!selectorActive && targetStatus.enabled) { + throw new Error('candidate cell is already enabled') + } else if ( + selectorActive && + !['migration-only', 'existing-only'].includes(targetAdmission) + ) { + throw new Error('candidate cell must not be generally admitted') + } + const capacity = await adminPost(config.directorOrigin, '/v1/admin/evacuation-capacity', { + v: 1, + sourceCellId: source.cellId, + targetCellId: target.cellId + }) + if (capacity.requiredTargetUnits > capacity.availableTargetUnits) { + throw new Error('candidate lacks survivor request-unit headroom') + } + deps.emit({ + event: 'candidate_preflight', + mode: config.mode, + sourceCellId: source.cellId, + targetCellId: target.cellId, + sourceOrigin: source.origin, + targetOrigin: target.origin, + sourceMig: source.migName, + targetMig: target.migName, + sourceBackend: source.backendName, + targetBackend: target.backendName, + sourceDigest: source.imageDigest, + targetDigest: target.imageDigest, + sourceAssignments: capacity.sourceAssignments, + requiredTargetUnits: capacity.requiredTargetUnits, + availableTargetUnits: capacity.availableTargetUnits + }) + if (config.mode === 'preflight') return + if (config.mode === 'enable-empty-cell') { + await setCellState(config, adminPost, target.cellId, true) + deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: true }) + return + } + // Fresh execution starts from source-only admission; continuation preserves its target. + if ( + !continuingEvacuation && + (selectorActive ? sourceAdmission !== 'existing-only' : !sourceStatus.enabled) + ) { + throw new Error( + selectorActive ? 'source cell is not existing-only' : 'source cell is not enabled' + ) + } + if (selectorActive && targetAdmission !== 'migration-only') { + throw new Error('target cell is not migration-only') + } + + let migrationsStarted = 0 + try { + if (!selectorActive) { + if (sourceStatus.enabled) await setCellState(config, adminPost, source.cellId, false) + if (!targetStatus.enabled) await setCellState(config, adminPost, target.cellId, true) + } + for (;;) { + const result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', { + v: 1, + sourceCellId: source.cellId, + targetCellId: target.cellId, + limit: config.batchSize + }) + migrationsStarted += result.started + deps.emit({ event: 'migration_batch', started: result.started, totalStarted: migrationsStarted }) + if (result.started === 0) break + } + } catch (error) { + await recoverCandidateFailure( + config, + deps, + adminPost, + source, + target, + !continuingEvacuation, + selectorActive + ) + throw error + } + + try { + if (selectorActive) { + const currentSelector = await inspectAdmissionSelector(selectorPost) + if ( + currentSelector.selector.generation !== selectorInspection.selector.generation || + JSON.stringify(currentSelector.selector.membership) !== + JSON.stringify(selectorInspection.selector.membership) + ) { + throw new Error('admission selector changed before drain') + } + } + await drainSource(config, deps, token, source) + await waitForMigration(config, deps, adminPost, false) + const completion = await waitForMigration(config, deps, adminPost, true) + if (completion.inProgress > 0) { + throw new Error('candidate migration remains pending for inactive target controls') + } + } catch (error) { + // A completion response can be lost after its transaction commits. Never + // reverse admission here merely because no in-progress row remains. + await recoverCandidateFailure( + config, + deps, + adminPost, + source, + target, + false, + selectorActive + ) + throw error + } + + await verifyCandidateCompletion(config, adminPost, source, target, deps.emit) +} + +export function aggregateCellStatus(status) { + return { + cellId: status.cellId, + enabled: status.enabled, + assignments: status.assignments, + activityLeases: status.activityLeases, + activityRequestUnits: status.activityRequestUnits, + reservedRequests: status.reservedRequests, + outgoingMigrations: status.outgoingMigrations, + incomingMigrations: status.incomingMigrations, + runtimeReady: status.runtime?.ready ?? false, + heartbeatFresh: status.runtime?.heartbeatFresh ?? false, + observedRequests: status.runtime?.observedRequests ?? null + } +} + +export async function main(argv = process.argv.slice(2)) { + await runCandidateDeployment(parseArguments(argv)) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs new file mode 100644 index 00000000000..3eccd1cf191 --- /dev/null +++ b/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs @@ -0,0 +1,889 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { + assertDeploymentConnectionCapacity, + createAdminPost, + deployment, + parseArguments, + runCandidateDeployment, + selectDeployments, + suppliedAdminIdentityToken, + validateBackend, + validateInstance, + validateMig +} from './deploy-relay-gce-candidate.mjs' + +const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com' +const digestA = `sha256:${'a'.repeat(64)}` +const digestB = `sha256:${'b'.repeat(64)}` + +test('retries evacuation-status HTTP 500 without widening other admin retries', async () => { + const events = [] + let calls = 0 + const adminPost = createAdminPost({}, { + fetch: async () => { + calls++ + return calls === 1 + ? new Response(JSON.stringify({ error: 'transient' }), { status: 500 }) + : new Response(JSON.stringify({ ok: true }), { status: 200 }) + }, + emit: (event) => events.push(event), + wait: async () => {}, + random: () => 0 + }, 'token') + assert.deepEqual( + await adminPost('https://relay.example', '/v1/admin/evacuation-status', {}), + { ok: true } + ) + assert.equal(calls, 2) + assert.deepEqual(events, [{ + event: 'candidate_admin_retry', + path: '/v1/admin/evacuation-status', + attempt: 1, + reason: 'http_500' + }]) + + calls = 0 + await assert.rejects( + createAdminPost({}, { + fetch: async () => { + calls++ + return new Response(JSON.stringify({ error: 'persistent' }), { status: 500 }) + }, + emit: () => {}, + wait: async () => {}, + random: () => 0 + }, 'token')('https://relay.example', '/v1/admin/runtime-status', {}), + /runtime-status failed: persistent/ + ) + assert.equal(calls, 1) +}) + +test('verifies a 1,000-cap cell against both runtime and director telemetry', () => { + const expected = deployment( + { + ...topology().target, + connection_hard_cap: 1_000, + connection_unobserved_bound: 60 + }, + 'target' + ) + const capacity = { + hardCap: 1_000, + controlRebindReserve: 100, + ordinaryConnectionLimit: 900, + unobservedBound: 60, + normalAdmissionPause: 840 + } + assert.doesNotThrow(() => + assertDeploymentConnectionCapacity(expected, capacity, { + ...capacity, + heartbeatFresh: true + }) + ) + assert.throws( + () => + assertDeploymentConnectionCapacity(expected, capacity, { + ...capacity, + hardCap: 600, + heartbeatFresh: true + }), + /differs from Terraform/ + ) +}) + +test('verifies a 3,000-cap regional cell with a 2,840 placement boundary', () => { + const expected = deployment( + { + ...topology().target, + connection_hard_cap: 3_000, + connection_unobserved_bound: 60 + }, + 'target' + ) + const capacity = { + hardCap: 3_000, + controlRebindReserve: 100, + ordinaryConnectionLimit: 2_900, + unobservedBound: 60, + normalAdmissionPause: 2_840 + } + + assert.doesNotThrow(() => + assertDeploymentConnectionCapacity(expected, capacity, { + ...capacity, + heartbeatFresh: true + }) + ) +}) + +function topology() { + return { + source: { + origin: 'https://c1.relay.example.com', + zone: 'us-central1-b', + mig_name: 'relay-c1', + instance_group: 'https://compute.example/instanceGroups/relay-c1', + backend_name: 'relay-c1', + backend_id: 'https://compute.example/backendServices/relay-c1', + image: `us-central1-docker.pkg.dev/project/repo/relay@${digestA}`, + capacity_requests: 4_000, + initially_enabled: true + }, + target: { + origin: 'https://c2.relay.example.com', + zone: 'us-central1-c', + mig_name: 'relay-c2', + instance_group: 'https://compute.example/instanceGroups/relay-c2', + backend_name: 'relay-c2', + backend_id: 'https://compute.example/backendServices/relay-c2', + image: `us-central1-docker.pkg.dev/project/repo/relay@${digestB}`, + capacity_requests: 4_000, + initially_enabled: false + } + } +} + +function withTopology(operation) { + const directory = mkdtempSync(join(tmpdir(), 'relay-gce-candidate-')) + const file = join(directory, 'topology.json') + writeFileSync(file, JSON.stringify(topology())) + return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true })) +} + +function config(topologyFile, mode = 'preflight') { + return { + project: 'test-project', + directorOrigin: 'https://relay.example.com', + adminAudience: 'https://relay.example.com/v1/admin/drain', + topologyFile, + sourceCellId: 'source', + targetCellId: 'target', + runtimeServiceAccount, + mode, + batchSize: 100, + drainGraceMs: 120_000, + pollIntervalMs: 1, + timeoutMs: 1_000 + } +} + +function response(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +function migrationResponse(body) { + return response({ + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 0, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0, + ...body + }) +} + +function fakeCommand(args) { + const name = args[args.indexOf('describe') + 1] ?? args[args.indexOf('list-instances') + 1] + if (args.includes('list-instances')) { + return [ + { + instance: `https://compute.example/instances/${name}-vm`, + instanceStatus: 'RUNNING', + currentAction: 'NONE' + } + ] + } + if (args.includes('instance-groups')) { + return { + targetSize: 1, + updatePolicy: { + replacementMethod: 'RECREATE', + maxSurge: { fixed: 0 }, + maxUnavailable: { fixed: 1 } + } + } + } + if (args.includes('instances')) { + return { + networkInterfaces: [{ networkIP: '10.42.0.2' }], + serviceAccounts: [{ email: runtimeServiceAccount }] + } + } + const cell = name === 'relay-c1' ? topology().source : topology().target + return { protocol: 'HTTP', timeoutSec: 86_400, backends: [{ group: cell.instance_group }] } +} + +function harness({ + failTargetEnable = false, + failDrain = false, + failAfterRegistration = false, + failCompletionResponse = false, + sourceEnabled = true, + targetEnabled = false, + targetAssignments = 0, + migrationInProgress = 0, + migrationTargetRegistered = 0, + migrationTargetInactive = 0, + transientCompletionFailures = 0, + dormantSourceAssignments = 0, + selectorGeneration = 0 +} = {}) { + const state = { + source: { + enabled: selectorGeneration > 0 ? false : sourceEnabled, + assignments: 2, + activityLeases: 2 + }, + target: { + enabled: selectorGeneration > 0 ? true : targetEnabled, + assignments: targetAssignments, + activityLeases: targetAssignments + } + } + const events = [] + const stateChanges = [] + let batch = 0 + let migrationCompleted = false + let remainingTransientCompletionFailures = transientCompletionFailures + const fetch = async (url, options = {}) => { + const parsed = new URL(url) + if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true }) + const body = JSON.parse(options.body ?? '{}') + if (parsed.pathname === '/v1/admin/runtime-status') { + const target = parsed.origin.includes('c2.') + return response({ + v: 1, + role: 'cell', + cellId: target ? 'target' : 'source', + cellUrl: parsed.origin, + imageDigest: target ? digestB : digestA + }) + } + if (parsed.pathname === '/v1/admin/admission-selector/status') { + return response({ + v: 1, + selector: { + generation: selectorGeneration, + attemptId: null, + membership: { + existingOnly: + selectorGeneration > 0 + ? ['source'] + : Object.keys(state).filter((cellId) => !state[cellId].enabled), + migrationOnly: selectorGeneration > 0 ? ['target'] : [], + general: + selectorGeneration > 0 + ? [] + : Object.keys(state).filter((cellId) => state[cellId].enabled) + } + }, + intent: null + }) + } + if (parsed.pathname === '/v1/admin/cell-status') { + const cell = state[body.cellId] + return response({ + v: 1, + status: { + cellId: body.cellId, + cellUrl: topology()[body.cellId].origin, + enabled: cell.enabled, + admissionState: + selectorGeneration > 0 + ? body.cellId === 'source' + ? 'existing-only' + : 'migration-only' + : cell.enabled + ? 'general' + : 'existing-only', + assignments: cell.assignments, + activityLeases: cell.activityLeases, + activityRequestUnits: cell.activityLeases, + reservedRequests: cell.activityLeases, + outgoingMigrations: 0, + incomingMigrations: 0, + runtime: { + cellUrl: topology()[body.cellId].origin, + ready: true, + heartbeatFresh: true, + observedRequests: cell.activityLeases + } + } + }) + } + if (parsed.pathname === '/v1/admin/evacuation-capacity') { + return response({ sourceAssignments: 2, requiredTargetUnits: 4, availableTargetUnits: 4_000 }) + } + if (parsed.pathname === '/v1/admin/cell-state') { + stateChanges.push([body.cellId, body.enabled]) + if (failTargetEnable && body.cellId === 'target' && body.enabled) { + return response({ error: 'injected_enable_failure' }, 409) + } + state[body.cellId].enabled = body.enabled + return response({ ok: true }) + } + if (parsed.pathname === '/v1/admin/evacuate-cell') { + if (failAfterRegistration && batch > 0) { + return response({ error: 'injected_batch_failure' }, 503) + } + const started = batch++ === 0 ? 2 : 0 + return response({ v: 1, started }) + } + if (parsed.pathname === '/v1/admin/evacuation-status') { + if (failAfterRegistration) { + return migrationResponse({ + v: 1, + inProgress: 2, + targetRegistered: 1, + registeredSourceActive: 1, + completed: 0, + blocked: 1 + }) + } + if (failDrain) { + return migrationResponse({ + v: 1, + inProgress: 2, + targetRegistered: 0, + completed: 0, + blocked: 0 + }) + } + if (body.completeReady) { + state.source.assignments = dormantSourceAssignments + state.source.activityLeases = 0 + state.target.assignments = 2 + state.target.activityLeases = 2 + if (remainingTransientCompletionFailures > 0) { + remainingTransientCompletionFailures-- + throw new TypeError('injected transient fetch failure') + } + if (migrationTargetInactive > 0) { + return migrationResponse({ + v: 1, + inProgress: migrationTargetInactive, + targetRegistered: migrationTargetInactive, + registeredTargetInactive: migrationTargetInactive, + completed: 0, + blocked: migrationTargetInactive + }) + } + migrationCompleted = true + if (failCompletionResponse) { + return response({ error: 'injected_completion_response_failure' }, 503) + } + return migrationResponse({ + v: 1, + inProgress: 0, + targetRegistered: 0, + completed: 2, + blocked: 0 + }) + } + if (migrationCompleted) { + return migrationResponse({ + v: 1, + inProgress: 0, + targetRegistered: 0, + completed: 0, + blocked: 0 + }) + } + if (batch === 0) { + return migrationResponse({ + v: 1, + inProgress: migrationInProgress, + targetRegistered: migrationTargetRegistered, + completed: 0, + blocked: 0 + }) + } + return migrationResponse({ + v: 1, + inProgress: 2, + targetRegistered: 2, + registeredCompletable: 2, + completed: 0, + blocked: 0 + }) + } + if (parsed.pathname === '/v1/admin/drain') { + return failDrain ? response({ error: 'injected_drain_failure' }, 503) : response({ ok: true }) + } + return response({ error: 'unexpected_request' }, 500) + } + return { + overrides: { + commandJson: fakeCommand, + identityToken: () => 'secret-token-never-emitted', + fetch, + emit: (event) => events.push(event), + wait: async () => undefined, + random: () => 0 + }, + events, + stateChanges, + state + } +} + +test('requires explicit dry-run/execute inputs and a distinct disabled candidate', () => { + assert.throws(() => parseArguments([]), /missing --project/) + assert.throws( + () => + parseArguments([ + '--project', + 'project', + '--director-origin', + 'https://relay.example.com', + '--admin-audience', + 'https://relay.example.com/not-drain', + '--topology-file', + 'topology.json', + '--source-cell-id', + 'source', + '--target-cell-id', + 'target', + '--runtime-service-account', + runtimeServiceAccount, + '--mode', + 'preflight' + ]), + /director drain URL/ + ) + assert.throws(() => selectDeployments(topology(), 'source', 'source'), /must differ/) + const overlapping = topology() + overlapping.target.backend_id = overlapping.source.backend_id + assert.throws(() => selectDeployments(overlapping, 'source', 'target'), /backendId overlap/) + const enabled = topology() + enabled.target.initially_enabled = true + assert.throws(() => selectDeployments(enabled, 'source', 'target'), /initially disabled/) +}) + +test('accepts only a bounded JWT-shaped supplied admin identity token', () => { + assert.equal(suppliedAdminIdentityToken({}), null) + assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc') + assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' })) + assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' })) + assert.throws(() => + suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` }) + ) +}) + +test('rejects unsafe fixed-one topology, public IPs, and backend overlap', () => { + const expected = selectDeployments(topology(), 'source', 'target').target + assert.throws(() => validateMig({ targetSize: 2 }, [], expected), /fixed-one/) + assert.throws( + () => + validateInstance( + { + networkInterfaces: [{ accessConfigs: [{ natIP: '203.0.113.1' }] }], + serviceAccounts: [{ email: runtimeServiceAccount }] + }, + expected, + runtimeServiceAccount + ), + /public IP/ + ) + assert.throws( + () => validateBackend({ protocol: 'HTTP', timeoutSec: 86_400, backends: [] }, expected), + /topology mismatch/ + ) +}) + +test('preflights exact served digests and survivor headroom without mutating admission', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness() + await runCandidateDeployment(config(file), overrides) + assert.deepEqual(stateChanges, []) + assert.equal(events[0].event, 'candidate_preflight') + assert.equal(events[0].targetDigest, digestB) + assert.equal(JSON.stringify(events).includes('secret-token'), false) + }) +}) + +test('audits a partially committed migration without changing admission or completing rows', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + sourceEnabled: false, + targetEnabled: true, + targetAssignments: 1, + migrationInProgress: 2, + migrationTargetRegistered: 2 + }) + await runCandidateDeployment(config(file, 'audit'), overrides) + assert.deepEqual(stateChanges, []) + assert.deepEqual(events, [ + { + event: 'candidate_audit', + source: { + cellId: 'source', + enabled: false, + assignments: 2, + activityLeases: 2, + activityRequestUnits: 2, + reservedRequests: 2, + outgoingMigrations: 0, + incomingMigrations: 0, + runtimeReady: true, + heartbeatFresh: true, + observedRequests: 2 + }, + target: { + cellId: 'target', + enabled: true, + assignments: 1, + activityLeases: 1, + activityRequestUnits: 1, + reservedRequests: 1, + outgoingMigrations: 0, + incomingMigrations: 0, + runtimeReady: true, + heartbeatFresh: true, + observedRequests: 1 + }, + migration: { + v: 1, + inProgress: 2, + targetRegistered: 2, + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 0, + blocked: 0, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 + } + } + ]) + }) +}) + +test('preflights with source admission disabled but still refuses execution', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ sourceEnabled: false }) + await runCandidateDeployment(config(file), overrides) + assert.deepEqual(stateChanges, []) + assert.equal(events.at(-1).event, 'candidate_preflight') + await assert.rejects( + runCandidateDeployment(config(file, 'execute'), overrides), + /source cell is not enabled/ + ) + assert.deepEqual(stateChanges, []) + }) +}) + +test('explicitly resets only an empty declared candidate to disabled admission', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + sourceEnabled: false, + targetEnabled: true + }) + await runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides) + assert.deepEqual(stateChanges, [['target', false]]) + assert.deepEqual(events.at(-1), { + event: 'candidate_admission_reset', + targetCellId: 'target', + changed: true + }) + }) +}) + +test('refuses to reset candidate admission while it owns durable activity', async () => { + await withTopology(async (file) => { + const { overrides, stateChanges } = harness({ targetEnabled: true, targetAssignments: 1 }) + await assert.rejects( + runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides), + /requires zero durable activity/ + ) + assert.deepEqual(stateChanges, []) + }) +}) + +test('disables only new admission while preserving durable candidate activity', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + targetEnabled: true, + targetAssignments: 1 + }) + await runCandidateDeployment(config(file, 'disable-cell'), overrides) + assert.deepEqual(stateChanges, [['target', false]]) + assert.deepEqual(events.at(-1), { + event: 'cell_admission_disabled', + targetCellId: 'target', + changed: true, + assignments: 1, + activityLeases: 1, + reservedRequests: 1, + outgoingMigrations: 0, + incomingMigrations: 0 + }) + }) +}) + +test('explicitly enables only an empty preflighted cell for admission', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ sourceEnabled: false }) + await runCandidateDeployment(config(file, 'enable-empty-cell'), overrides) + assert.deepEqual(stateChanges, [['target', true]]) + assert.equal(events.at(-2).event, 'candidate_preflight') + assert.deepEqual(events.at(-1), { + event: 'cell_admission_enabled', + targetCellId: 'target', + changed: true + }) + }) +}) + +test('refuses to enable cell admission while it owns durable activity', async () => { + await withTopology(async (file) => { + const { overrides, stateChanges } = harness({ targetAssignments: 1 }) + await assert.rejects( + runCandidateDeployment(config(file, 'enable-empty-cell'), overrides), + /requires zero durable activity/ + ) + assert.deepEqual(stateChanges, []) + }) +}) + +test('executes target-first evacuation and verifies aggregate drained counts', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness() + await runCandidateDeployment(config(file, 'execute'), overrides) + assert.deepEqual(stateChanges.slice(0, 2), [ + ['source', false], + ['target', true] + ]) + assert.equal(events.at(-1).event, 'candidate_complete') + assert.equal(events.at(-1).targetAssignments, 2) + }) +}) + +test('executes within selector membership without legacy admission writes', async () => { + await withTopology(async (file) => { + const testHarness = harness({ selectorGeneration: 1 }) + await runCandidateDeployment(config(file, 'execute'), testHarness.overrides) + assert.deepEqual(testHarness.stateChanges, []) + assert.equal(testHarness.state.source.enabled, false) + assert.equal(testHarness.state.target.enabled, true) + }) +}) + +test('never restores legacy general admission after selector-era failure', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + selectorGeneration: 1, + failAfterRegistration: true + }) + await assert.rejects( + runCandidateDeployment(config(file, 'execute'), testHarness.overrides), + /injected_batch_failure/ + ) + assert.deepEqual(testHarness.stateChanges, []) + assert.equal(testHarness.state.source.enabled, false) + }) +}) + +test('continues a partial evacuation without resetting target admission', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + sourceEnabled: true, + targetEnabled: true, + targetAssignments: 1, + migrationInProgress: 1, + migrationTargetRegistered: 1 + }) + await runCandidateDeployment(config(file, 'continue-evacuation'), overrides) + assert.deepEqual(stateChanges, [['source', false]]) + assert.equal(events.at(-1).event, 'candidate_complete') + }) +}) + +test('refuses continued evacuation unless target admission is already enabled', async () => { + await withTopology(async (file) => { + const { overrides, stateChanges } = harness() + await assert.rejects( + runCandidateDeployment(config(file, 'continue-evacuation'), overrides), + /requires enabled target/ + ) + assert.deepEqual(stateChanges, []) + }) +}) + +test('preserves partial target admission when continued batching fails', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + failAfterRegistration: true, + sourceEnabled: true, + targetEnabled: true, + targetAssignments: 1, + migrationInProgress: 1, + migrationTargetRegistered: 1 + }) + await assert.rejects( + runCandidateDeployment(config(file, 'continue-evacuation'), overrides), + /injected_batch_failure/ + ) + assert.deepEqual(stateChanges, [['source', false]]) + assert.equal(events.at(-1).event, 'candidate_forward_recovery_required') + }) +}) + +test('resumes only a committed forward migration and preserves dormant source assignments', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + sourceEnabled: false, + targetEnabled: true, + migrationInProgress: 2, + migrationTargetRegistered: 2, + dormantSourceAssignments: 7 + }) + await runCandidateDeployment(config(file, 'recover-forward'), overrides) + assert.deepEqual(stateChanges, []) + assert.deepEqual(events.at(-1), { + event: 'candidate_forward_recovered', + sourceCellId: 'source', + targetCellId: 'target', + dormantSourceAssignments: 7, + targetAssignments: 2, + targetActivityLeases: 2, + targetReservedRequests: 2 + }) + }) +}) + +test('retries a transient idempotent completion request without reversing admission', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + sourceEnabled: false, + targetEnabled: true, + migrationInProgress: 2, + migrationTargetRegistered: 2, + transientCompletionFailures: 1 + }) + await runCandidateDeployment(config(file, 'recover-forward'), overrides) + assert.deepEqual(stateChanges, []) + assert.deepEqual( + events.filter(({ event }) => event === 'candidate_admin_retry'), + [ + { + event: 'candidate_admin_retry', + path: '/v1/admin/evacuation-status', + attempt: 1, + reason: 'transport' + } + ] + ) + assert.equal(events.at(-1).event, 'candidate_forward_recovered') + }) +}) + +test('stops forward recovery promptly when only registered offline targets remain', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ + sourceEnabled: false, + targetEnabled: true, + migrationInProgress: 2, + migrationTargetRegistered: 2, + migrationTargetInactive: 2 + }) + await assert.rejects( + runCandidateDeployment(config(file, 'recover-forward'), overrides), + /pending for inactive target controls/ + ) + assert.deepEqual(stateChanges, []) + assert.deepEqual(events.at(-1), { + event: 'candidate_forward_pending', + sourceCellId: 'source', + targetCellId: 'target', + inProgress: 2, + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 2 + }) + }) +}) + +test('refuses forward recovery unless source and target admission match committed direction', async () => { + await withTopology(async (file) => { + const { overrides, stateChanges } = harness() + await assert.rejects( + runCandidateDeployment(config(file, 'recover-forward'), overrides), + /requires disabled source and enabled target/ + ) + assert.deepEqual(stateChanges, []) + }) +}) + +test('re-enables an intact source when candidate admission fails before migration', async () => { + await withTopology(async (file) => { + const { overrides, stateChanges } = harness({ failTargetEnable: true }) + await assert.rejects( + runCandidateDeployment(config(file, 'execute'), overrides), + /injected_enable_failure/ + ) + assert.deepEqual(stateChanges, [ + ['source', false], + ['target', true], + ['source', true], + ['target', false] + ]) + }) +}) + +test('re-enables the source and waits for lease rollback when no target registered', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ failDrain: true }) + await assert.rejects( + runCandidateDeployment(config(file, 'execute'), overrides), + /injected_drain_failure/ + ) + assert.deepEqual(stateChanges.slice(-1), [['source', true]]) + assert.equal(events.at(-1).event, 'candidate_rollback_waiting_for_lease_expiry') + }) +}) + +test('preserves both routes for forward recovery after a target registration', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ failAfterRegistration: true }) + await assert.rejects( + runCandidateDeployment(config(file, 'execute'), overrides), + /injected_batch_failure/ + ) + assert.deepEqual(stateChanges, [ + ['source', false], + ['target', true] + ]) + assert.equal(events.at(-1).event, 'candidate_forward_recovery_required') + assert.equal(events.at(-1).targetRegistered, 1) + }) +}) + +test('does not reverse admission after completion commits but its response is lost', async () => { + await withTopology(async (file) => { + const { overrides, events, stateChanges } = harness({ failCompletionResponse: true }) + await assert.rejects( + runCandidateDeployment(config(file, 'execute'), overrides), + /injected_completion_response_failure/ + ) + assert.deepEqual(stateChanges, [ + ['source', false], + ['target', true] + ]) + assert.equal(events.at(-1).event, 'candidate_forward_recovery_required') + assert.equal(events.at(-1).targetRegistered, 0) + }) +}) diff --git a/cloud/dev/scripts/deploy-relay-gce-multi-target.mjs b/cloud/dev/scripts/deploy-relay-gce-multi-target.mjs new file mode 100644 index 00000000000..e36570947ad --- /dev/null +++ b/cloud/dev/scripts/deploy-relay-gce-multi-target.mjs @@ -0,0 +1,3135 @@ +import { readFileSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { resolve4 } from 'node:dns/promises' +import { pathToFileURL } from 'node:url' +import { + aggregateCellStatus, + assertDeploymentConnectionCapacity, + createAdminPost, + defaultCommandJson, + defaultIdentityToken, + deployment, + drainSource, + inspectCell, + setCellState, + suppliedFenceMutationIdentityToken, + validateBackend, + validateInstance, + validateMig +} from './deploy-relay-gce-candidate.mjs' +import { + abortSupersededTerraformFenceBeforeUpload, + abortTerraformFenceBeforeApply, + adoptLegacyTerraformFence, + assertTerraformFenceSet, + assertTerraformFenceZeroDiff, + deleteTerraformFencePlan, + downloadTerraformFencePlan, + inspectCompletedTerraformFenceProgress, + inspectTerraformFenceProgress, + assertTerraformFenceStateFenced, + readTerraformStateObjectBinding, + recoverSupersededCompletedTerraformFence, + resolveTerraformFencePlanGeneration, + resumeTerraformFence, + runTerraformFenceApply, + uploadTerraformFencePlan +} from './relay-gce-terraform-fence.mjs' +import { + addExactMigrationCells, + applyExactAdmissionSelector, + inspectAdmissionSelector, + membershipWithStates, + selectorCellState +} from './relay-admission-selector.mjs' + +const DEFAULT_BATCH_SIZE = 100 +const DEFAULT_CONNECTION_CEILING = 600 +const DEFAULT_MINIMUM_LEASE_MS = 10 * 60 * 1_000 +const DEFAULT_POLL_MS = 5_000 +const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000 +const CUTOVER_CONNECTION_HARD_CAP = 600 +const CUTOVER_CONTROL_REBIND_RESERVE = 100 +const MAX_PRE_AUTH_CONNECTIONS = 45 +const SELECTOR_ROLLBACK_TAG = 'selector-rollback' +const SELECTOR_REVISION_MARKER = '3' +const FENCE_BROKER_MUTATION_ROUTES = new Set([ + '/v1/admin/cell-fence-adopt-legacy', + '/v1/admin/cell-fence-commit-legacy-adoption', + '/v1/admin/cell-fence-attest', + '/v1/admin/cell-fence-attempt-prepare', + '/v1/admin/cell-fence-attempt-start', + '/v1/admin/cell-fence-attempt-plan', + '/v1/admin/cell-fence-attempt-operation', + '/v1/admin/cell-fence-attempt-abort', + '/v1/admin/migration-supersede-cell' +]) + +function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) { + throw new Error(`${name} must be a positive integer`) + } + return parsed +} + +function nonnegativeInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) { + const parsed = Number(value) + if (!Number.isSafeInteger(parsed) || parsed < 0 || parsed > maximum) { + throw new Error(`${name} must be a nonnegative integer`) + } + return parsed +} + +function canonicalOrigin(value, name) { + const url = new URL(value) + if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') { + throw new Error(`${name} must be a canonical HTTPS origin`) + } + return value +} + +function targetIds(value, minimum) { + const ids = [...new Set(value.split(',').map((item) => item.trim()).filter(Boolean))].sort() + if (ids.length < minimum || ids.some((id) => !/^[a-z][a-z0-9-]{0,127}$/.test(id))) { + throw new Error( + `--target-cell-ids must contain at least ${minimum} distinct cell ID${minimum === 1 ? '' : 's'}` + ) + } + return ids +} + +function optionalCellIds(value, name) { + const ids = [...new Set(String(value ?? '').split(',').map((item) => item.trim()).filter(Boolean))] + .sort() + if (ids.some((id) => !/^[a-z][a-z0-9-]{0,127}$/.test(id))) { + throw new Error(`${name} contains an invalid cell ID`) + } + return ids +} + +export function parseMultiTargetArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) + values[key.slice(2)] = value + } + for (const key of [ + 'project', + 'director-origin', + 'admin-audience', + 'topology-file', + 'source-cell-id', + 'target-cell-ids', + 'runtime-service-account', + 'mode' + ]) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if ( + ![ + 'audit', + 'preflight', + 'execute', + 'cutover-admission', + 'add-migration-cells', + 'promote-general-cell', + 'retire-migration-cell', + 'recover-forward', + 'fence-source', + 'abort-fence-source', + 'supersede-target' + ].includes(values.mode) + ) { + throw new Error( + '--mode must be audit, preflight, execute, cutover-admission, add-migration-cells, promote-general-cell, retire-migration-cell, recover-forward, fence-source, abort-fence-source, or supersede-target' + ) + } + const singleTargetMode = [ + 'add-migration-cells', + 'promote-general-cell', + 'retire-migration-cell' + ].includes(values.mode) + const parsedTargetIds = targetIds(values['target-cell-ids'], singleTargetMode ? 1 : 2) + const generalCellIds = optionalCellIds(values['general-cell-ids'], '--general-cell-ids') + const failedTargetCellId = values['failed-target-cell-id'] + const replacementTargetCellId = values['replacement-target-cell-id'] + if ( + ['promote-general-cell', 'retire-migration-cell'].includes(values.mode) && + parsedTargetIds.length !== 1 + ) { + throw new Error(`${values.mode} requires exactly one target cell ID`) + } + if (values.mode === 'supersede-target') { + if (!failedTargetCellId || !replacementTargetCellId) { + throw new Error('supersede-target requires failed and replacement target cell IDs') + } + if ( + failedTargetCellId === replacementTargetCellId || + !parsedTargetIds.includes(failedTargetCellId) || + !parsedTargetIds.includes(replacementTargetCellId) || + parsedTargetIds.length !== 2 + ) { + throw new Error('supersede-target target set must exactly match failed and replacement') + } + } + const selectorMode = [ + 'cutover-admission', + 'add-migration-cells', + 'promote-general-cell', + 'retire-migration-cell' + ].includes(values.mode) + if (selectorMode) { + for (const key of ['director-region', 'director-service', 'director-min-instances']) { + if (!values[key]) throw new Error(`${values.mode} requires --${key}`) + } + if (values.mode === 'cutover-admission' && generalCellIds.length === 0) { + throw new Error('cutover-admission requires --general-cell-ids') + } + if ( + values['selector-attempt-id'] && + !/^[A-Za-z0-9_-]{8,128}$/.test(values['selector-attempt-id']) + ) { + throw new Error('--selector-attempt-id is invalid') + } + if ( + ['add-migration-cells', 'promote-general-cell', 'retire-migration-cell'].includes( + values.mode + ) && + !values['selector-attempt-id'] + ) { + throw new Error(`${values.mode} requires --selector-attempt-id`) + } + } + const capacityBoundModes = [ + 'cutover-admission', + 'add-migration-cells', + 'recover-forward', + 'fence-source' + ] + if ( + capacityBoundModes.includes(values.mode) && + !values['unobserved-connection-bound'] + ) { + throw new Error(`${values.mode} requires --unobserved-connection-bound`) + } + const adminAudience = new URL(values['admin-audience']) + if ( + adminAudience.protocol !== 'https:' || + adminAudience.pathname !== '/v1/admin/drain' || + adminAudience.search || + adminAudience.hash + ) { + throw new Error('--admin-audience must be the director drain URL') + } + if (!['staging', 'production'].includes(values.environment ?? 'production')) { + throw new Error('--environment must be staging or production') + } + if ( + ['fence-source', 'abort-fence-source', 'supersede-target'].includes(values.mode) && + !/^[a-f0-9]{40}$/.test(values['fence-commit'] ?? '') + ) { + throw new Error('Terraform fence modes require the exact --fence-commit') + } + const completedFenceFields = { + attemptId: values['completed-fence-attempt-id'], + fenceCommit: values['completed-fence-commit'], + gceOperation: values['completed-fence-operation'], + terraformStateSerial: values['completed-fence-state-serial'], + planObjectGeneration: values['completed-fence-plan-generation'], + terraformStateObjectGeneration: values['completed-fence-state-generation'], + terraformStateObjectSha256: values['completed-fence-state-sha256'], + principalEmail: values['fence-broker-service-account'] + } + const completedFenceValues = Object.values(completedFenceFields) + const completedFenceRecovery = + completedFenceValues.every((value) => value === undefined) + ? undefined + : completedFenceFields + if ( + completedFenceRecovery && + (values.mode !== 'supersede-target' || + completedFenceValues.some((value) => value === undefined) || + !/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test( + completedFenceRecovery.attemptId + ) || + !/^[a-f0-9]{40}$/.test(completedFenceRecovery.fenceCommit) || + completedFenceRecovery.fenceCommit === values['fence-commit'] || + !/^[A-Za-z0-9._-]{1,256}$/.test(completedFenceRecovery.gceOperation) || + !/^[1-9][0-9]{0,30}$/.test(completedFenceRecovery.planObjectGeneration) || + !/^[1-9][0-9]{0,30}$/.test( + completedFenceRecovery.terraformStateObjectGeneration + ) || + !/^[a-f0-9]{64}$/.test( + completedFenceRecovery.terraformStateObjectSha256 + ) || + !/^[^@\s]+@[^@\s]+\.gserviceaccount\.com$/.test( + completedFenceRecovery.principalEmail + )) + ) { + throw new Error('completed Terraform fence recovery inputs are invalid') + } + const minimumLeaseRemainingMs = positiveInteger( + values['minimum-lease-remaining-ms'] ?? DEFAULT_MINIMUM_LEASE_MS, + '--minimum-lease-remaining-ms', + 60 * 60 * 1_000 + ) + if (minimumLeaseRemainingMs < DEFAULT_MINIMUM_LEASE_MS) { + throw new Error('--minimum-lease-remaining-ms cannot be below 600000') + } + return { + project: values.project, + directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'), + adminAudience: adminAudience.toString(), + topologyFile: values['topology-file'], + sourceCellId: values['source-cell-id'], + targetCellIds: parsedTargetIds, + generalCellIds, + directorRegion: values['director-region'], + directorService: values['director-service'], + directorMinimumInstances: selectorMode + ? positiveInteger(values['director-min-instances'], '--director-min-instances', 1_000) + : undefined, + selectorAttemptId: values['selector-attempt-id'], + unobservedConnectionBound: + capacityBoundModes.includes(values.mode) + ? nonnegativeInteger( + values['unobserved-connection-bound'], + '--unobserved-connection-bound', + CUTOVER_CONNECTION_HARD_CAP - CUTOVER_CONTROL_REBIND_RESERVE - 1 + ) + : undefined, + failedTargetCellId, + replacementTargetCellId, + completedFenceRecovery: completedFenceRecovery + ? { + ...completedFenceRecovery, + terraformStateSerial: nonnegativeInteger( + completedFenceRecovery.terraformStateSerial, + '--completed-fence-state-serial' + ) + } + : undefined, + runtimeServiceAccount: values['runtime-service-account'], + environment: values.environment ?? 'production', + fenceCommit: values['fence-commit'], + terraformDir: values['terraform-dir'] ?? 'infra/terraform', + terraformVarFile: + values['terraform-var-file'] ?? + `environments/${values.environment ?? 'production'}.tfvars`, + mode: values.mode, + batchSize: positiveInteger(values['batch-size'] ?? DEFAULT_BATCH_SIZE, '--batch-size', 100), + connectionCeiling: positiveInteger( + values['connection-ceiling'] ?? DEFAULT_CONNECTION_CEILING, + '--connection-ceiling', + 100_000 + ), + minimumLeaseRemainingMs, + drainGraceMs: positiveInteger( + values['drain-grace-ms'] ?? 120_000, + '--drain-grace-ms', + 60 * 60 * 1_000 + ), + pollIntervalMs: positiveInteger( + values['poll-interval-ms'] ?? DEFAULT_POLL_MS, + '--poll-interval-ms', + 60_000 + ), + timeoutMs: positiveInteger( + values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS, + '--timeout-ms', + 60 * 60 * 1_000 + ) + } +} + +export function selectMultiTargetDeployments(topology, sourceCellId, targetCellIds) { + const legacyCapacityTopology = Object.values(topology).every( + (cell) => + cell && + typeof cell === 'object' && + !Object.hasOwn(cell, 'connection_hard_cap') && + !Object.hasOwn(cell, 'connection_unobserved_bound') + ) + const fromTopology = (cellId) => ({ + ...deployment(topology[cellId], cellId), + legacyCapacityTopology + }) + const source = fromTopology(sourceCellId) + const targets = targetCellIds.map(fromTopology) + const resources = new Map() + for (const cell of [source, ...targets]) { + for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) { + const resourceKey = `${key}:${cell[key]}` + const previous = resources.get(resourceKey) + if (previous) throw new Error(`${cell.cellId} ${key} overlaps ${previous}`) + resources.set(resourceKey, cell.cellId) + } + } + if (targets.some((target) => target.initiallyEnabled)) { + throw new Error('every target must be declared initially disabled') + } + return { source, targets } +} + +function revisionEnvironment(revision) { + return Object.fromEntries( + (revision.spec?.containers?.[0]?.env ?? []) + .filter((entry) => entry.name && 'value' in entry) + .map((entry) => [entry.name, entry.value]) + ) +} + +function revisionMinimum(revision) { + return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0) +} + +function directorInventory(environment, revisionName) { + let cells + try { + cells = JSON.parse(environment.ORCA_RELAY_CELLS_JSON) + } catch { + throw new Error(`${revisionName} has an invalid director inventory`) + } + const ids = Array.isArray(cells) ? cells.map((cell) => cell?.id) : [] + if ( + ids.length === 0 || + ids.some((id) => typeof id !== 'string' || id.length === 0) || + new Set(ids).size !== ids.length + ) { + throw new Error(`${revisionName} has an invalid director inventory`) + } + return JSON.stringify(cells) +} + +export function verifySelectorCompatibleDirector(config, deps) { + const common = [ + '--project', + config.project, + '--region', + config.directorRegion, + '--format=json' + ] + const service = deps.commandJson([ + 'run', + 'services', + 'describe', + config.directorService, + ...common + ]) + const active = (service.status?.traffic ?? []).filter( + (entry) => Number(entry.percent ?? 0) > 0 + ) + const rollback = (service.status?.traffic ?? []).find( + (entry) => entry.tag === SELECTOR_ROLLBACK_TAG + ) + if ( + active.length !== 1 || + Number(active[0].percent) !== 100 || + !active[0].revisionName || + !rollback?.revisionName || + Number(rollback.percent ?? 0) !== 0 || + rollback.revisionName === active[0].revisionName + ) { + throw new Error('director lacks an isolated compatible rollback revision') + } + const revisions = deps.commandJson([ + 'run', + 'revisions', + 'list', + '--service', + config.directorService, + ...common + ]) + const allowed = new Set([active[0].revisionName, rollback.revisionName]) + const names = revisions.map((revision) => revision.metadata?.name).filter(Boolean) + if ( + revisions.length !== 2 || + names.length !== 2 || + names.some((name) => !allowed.has(name)) + ) { + throw new Error('old or pre-selector director revisions still exist') + } + let compatibleImage + let compatibleInventory + for (const revisionName of allowed) { + const revision = deps.commandJson([ + 'run', + 'revisions', + 'describe', + revisionName, + ...common + ]) + const environment = revisionEnvironment(revision) + if ( + environment.ORCA_RELAY_ROLE !== 'director' || + environment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== SELECTOR_REVISION_MARKER + ) { + throw new Error(`${revisionName} is not selector-compatible`) + } + const inventory = directorInventory(environment, revisionName) + if (compatibleInventory && inventory !== compatibleInventory) { + throw new Error('active and rollback director inventories do not match') + } + compatibleInventory = inventory + const image = revision.spec?.containers?.[0]?.image + if (!image || (compatibleImage && image !== compatibleImage)) { + throw new Error('active and rollback director images do not match') + } + compatibleImage = image + if (revisionName === rollback.revisionName && revisionMinimum(revision) !== 0) { + throw new Error('selector rollback revision is not scale-to-zero') + } + if ( + revisionName === active[0].revisionName && + !(revisionMinimum(revision) >= config.directorMinimumInstances) + ) { + throw new Error('active selector revision is below the required floor') + } + } + return { + activeRevision: active[0].revisionName, + rollbackRevision: rollback.revisionName + } +} + +function verifyActiveSelectorDirector(config, deps, cellId) { + const common = [ + '--project', + config.project, + '--region', + config.directorRegion, + '--format=json' + ] + const service = deps.commandJson([ + 'run', + 'services', + 'describe', + config.directorService, + ...common + ]) + const active = (service.status?.traffic ?? []).filter( + (entry) => Number(entry.percent ?? 0) > 0 + ) + if ( + active.length !== 1 || + Number(active[0].percent) !== 100 || + !active[0].revisionName + ) { + throw new Error('director lacks one active selector revision') + } + const revision = deps.commandJson([ + 'run', + 'revisions', + 'describe', + active[0].revisionName, + ...common + ]) + const environment = revisionEnvironment(revision) + const inventory = JSON.parse(directorInventory(environment, active[0].revisionName)) + if ( + environment.ORCA_RELAY_ROLE !== 'director' || + environment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== SELECTOR_REVISION_MARKER || + !(revisionMinimum(revision) >= config.directorMinimumInstances) || + !inventory.some((cell) => cell.id === cellId) + ) { + throw new Error('active director is not compatible with the promoted cell') + } + return { activeRevision: active[0].revisionName } +} + +export function pruneIncompatibleDirectorRevisions(config, deps) { + const common = [ + '--project', + config.project, + '--region', + config.directorRegion, + '--format=json' + ] + const service = deps.commandJson([ + 'run', + 'services', + 'describe', + config.directorService, + ...common + ]) + const traffic = service.status?.traffic ?? [] + const active = traffic.filter((entry) => Number(entry.percent ?? 0) > 0) + const rollback = traffic.find((entry) => entry.tag === SELECTOR_ROLLBACK_TAG) + const unexpectedTags = traffic.filter( + (entry) => entry.tag && entry.tag !== SELECTOR_ROLLBACK_TAG + ) + if ( + active.length !== 1 || + Number(active[0].percent) !== 100 || + !active[0].revisionName || + !rollback?.revisionName || + Number(rollback.percent ?? 0) !== 0 || + rollback.revisionName === active[0].revisionName || + unexpectedTags.length > 0 + ) { + throw new Error('director traffic is not ready for selector cutover') + } + const activeRevision = deps.commandJson([ + 'run', + 'revisions', + 'describe', + active[0].revisionName, + ...common + ]) + const rollbackRevision = deps.commandJson([ + 'run', + 'revisions', + 'describe', + rollback.revisionName, + ...common + ]) + const activeEnvironment = revisionEnvironment(activeRevision) + const rollbackEnvironment = revisionEnvironment(rollbackRevision) + const activeInventory = directorInventory(activeEnvironment, active[0].revisionName) + const rollbackInventory = directorInventory(rollbackEnvironment, rollback.revisionName) + if ( + activeEnvironment.ORCA_RELAY_ROLE !== 'director' || + rollbackEnvironment.ORCA_RELAY_ROLE !== 'director' || + activeEnvironment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== + SELECTOR_REVISION_MARKER || + rollbackEnvironment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== + SELECTOR_REVISION_MARKER || + !activeRevision.spec?.containers?.[0]?.image || + activeRevision.spec?.containers?.[0]?.image !== + rollbackRevision.spec?.containers?.[0]?.image || + activeInventory !== rollbackInventory || + revisionMinimum(rollbackRevision) !== 0 || + !(revisionMinimum(activeRevision) >= config.directorMinimumInstances) + ) { + throw new Error('director compatibility pair failed before revision pruning') + } + const retained = new Set([active[0].revisionName, rollback.revisionName]) + const revisions = deps.commandJson([ + 'run', + 'revisions', + 'list', + '--service', + config.directorService, + ...common + ]) + for (const revision of revisions) { + const revisionName = revision.metadata?.name + if (!revisionName) throw new Error('director revision list contains an unnamed revision') + if (retained.has(revisionName)) continue + deps.command([ + 'run', + 'revisions', + 'delete', + revisionName, + '--project', + config.project, + '--region', + config.directorRegion, + '--quiet' + ]) + } +} + +export function cutoverMembership(topology, config) { + const all = Object.keys(topology).sort() + const migration = new Set(config.targetCellIds) + const general = new Set(config.generalCellIds) + if ([...migration].some((cellId) => general.has(cellId))) { + throw new Error('general and migration-only cell sets overlap') + } + if (migration.has(config.sourceCellId) || general.has(config.sourceCellId)) { + throw new Error('cutover source must remain existing-only') + } + for (const cellId of [...migration, ...general]) { + if (!all.includes(cellId)) throw new Error(`selector cell ${cellId} is absent from topology`) + } + return { + existingOnly: all.filter((cellId) => !migration.has(cellId) && !general.has(cellId)), + migrationOnly: [...migration].sort(), + general: [...general].sort() + } +} + +function assertDistinctCutoverResources(cells) { + const resources = new Map() + for (const cell of cells) { + for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) { + const resourceKey = `${key}:${cell[key]}` + const previous = resources.get(resourceKey) + if (previous) throw new Error(`${cell.cellId} ${key} overlaps ${previous}`) + resources.set(resourceKey, cell.cellId) + } + } +} + +export function assertCutoverCellReady(cellId, status, unobservedConnectionBound) { + const capacity = status.runtimeConnectionCapacity + const directorCapacity = status.connectionCapacity + const process = status.process + const expectedPause = + CUTOVER_CONNECTION_HARD_CAP - + CUTOVER_CONTROL_REBIND_RESERVE - + unobservedConnectionBound + if ( + !capacity || + capacity.hardCap !== CUTOVER_CONNECTION_HARD_CAP || + capacity.controlRebindReserve !== CUTOVER_CONTROL_REBIND_RESERVE || + capacity.ordinaryConnectionLimit !== + CUTOVER_CONNECTION_HARD_CAP - CUTOVER_CONTROL_REBIND_RESERVE || + capacity.unobservedBound !== unobservedConnectionBound || + capacity.normalAdmissionPause !== expectedPause || + !directorCapacity || + directorCapacity.hardCap !== capacity.hardCap || + directorCapacity.controlRebindReserve !== capacity.controlRebindReserve || + directorCapacity.ordinaryConnectionLimit !== capacity.ordinaryConnectionLimit || + directorCapacity.unobservedBound !== capacity.unobservedBound || + directorCapacity.normalAdmissionPause !== capacity.normalAdmissionPause || + directorCapacity.heartbeatFresh !== true || + expectedPause <= 0 + ) { + throw new Error(`${cellId} does not expose the reviewed connection-capacity policy`) + } + if ( + !process || + !Number.isSafeInteger(process.enforcedConnectionUnits) || + process.enforcedConnectionUnits < 0 || + !Number.isSafeInteger(process.preAuthConnections) || + process.preAuthConnections < 0 || + !Number.isSafeInteger(directorCapacity.pendingControlReservations) || + directorCapacity.pendingControlReservations < 0 + ) { + throw new Error(`${cellId} has incomplete connection-capacity evidence`) + } + if (process.preAuthConnections >= MAX_PRE_AUTH_CONNECTIONS) { + throw new Error(`${cellId} has insufficient pre-auth connection headroom`) + } + const committedUnits = + process.enforcedConnectionUnits + directorCapacity.pendingControlReservations + if (!Number.isSafeInteger(committedUnits) || committedUnits >= expectedPause) { + throw new Error(`${cellId} has insufficient normal-admission connection headroom`) + } + if (status.draining) throw new Error(`${cellId} is draining before selector cutover`) +} + +async function inspectCutoverCells(topology, config, deps, adminPost, membership) { + const selectedIds = [...membership.migrationOnly, ...membership.general] + const selected = selectedIds.map((cellId) => deployment(topology[cellId], cellId)) + assertDistinctCutoverResources([ + deployment(topology[config.sourceCellId], config.sourceCellId), + ...selected + ]) + for (const cell of selected) { + const status = await inspectCell(config, deps, adminPost, cell) + assertCutoverCellReady(cell.cellId, status, config.unobservedConnectionBound) + } +} + +function projection(total, quota, assignments) { + return assignments === 0 ? 0 : Math.ceil((total * quota) / assignments) +} + +function connectionProjection(current, sourceConnections, sourceAssignments, quota) { + const unboundConnections = Math.max(0, sourceConnections - sourceAssignments) + return current + quota + unboundConnections +} + +function targetConnectionCeiling(config, target) { + return Math.min( + config.connectionCeiling, + target.connectionHardCap ?? CUTOVER_CONNECTION_HARD_CAP + ) +} + +function connectionReservationHeadroom(status, cellId) { + const capacity = status.connectionCapacity + const values = [ + capacity?.hardCap, + capacity?.controlRebindReserve, + capacity?.ordinaryConnectionLimit, + capacity?.unobservedBound, + capacity?.normalAdmissionPause, + capacity?.enforcedConnectionUnits, + capacity?.pendingControlReservations + ] + if ( + capacity?.heartbeatFresh !== true || + values.some((value) => !Number.isSafeInteger(value) || value < 0) || + capacity.ordinaryConnectionLimit !== capacity.hardCap - capacity.controlRebindReserve || + capacity.normalAdmissionPause !== + capacity.ordinaryConnectionLimit - capacity.unobservedBound + ) { + throw new Error(`${cellId} has inconsistent connection-reservation capacity`) + } + return Math.max( + 0, + capacity.normalAdmissionPause - + capacity.enforcedConnectionUnits - + capacity.pendingControlReservations + ) +} + +export function allocateTargetQuotas({ + sourceAssignments, + sourceConnections, + requiredTargetUnits, + targets, + connectionCeiling +}) { + const quotas = new Map(targets.map((target) => [target.cellId, 0])) + for (let assigned = 0; assigned < sourceAssignments; assigned++) { + const candidates = targets + .map((target) => { + const quota = quotas.get(target.cellId) + 1 + const projectedConnections = connectionProjection( + target.currentConnections, + sourceConnections, + sourceAssignments, + quota + ) + const projectedUnits = projection(requiredTargetUnits, quota, sourceAssignments) + return { target, quota, projectedConnections, projectedUnits } + }) + .filter( + ({ target, quota, projectedConnections, projectedUnits }) => + projectedConnections < (target.connectionCeiling ?? connectionCeiling) && + quota <= target.availableConnectionReservations && + projectedUnits <= target.availableTargetUnits + ) + .sort( + (left, right) => + left.projectedConnections - right.projectedConnections || + left.target.cellId.localeCompare(right.target.cellId) + ) + const selected = candidates[0] + if (!selected) throw new Error('multi-target connection or request-unit headroom exhausted') + quotas.set(selected.target.cellId, selected.quota) + } + return targets.map((target) => ({ + ...target, + quota: quotas.get(target.cellId), + projectedConnections: connectionProjection( + target.currentConnections, + sourceConnections, + sourceAssignments, + quotas.get(target.cellId) + ), + projectedUnits: projection( + requiredTargetUnits, + quotas.get(target.cellId), + sourceAssignments + ) + })) +} + +function defaultCommand(args) { + const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) + if (result.status !== 0) { + throw new Error(`gcloud ${args.slice(0, 5).join(' ')} failed: ${result.stderr.trim()}`) + } +} + +function defaultCommandResult(args) { + const result = spawnSync('gcloud', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'] + }) + return { status: result.status, stdout: result.stdout, stderr: result.stderr } +} + +function validatedProcessCounts(counts, cellId) { + for (const field of [ + 'totalConnections', + 'preAuthConnections', + 'controls', + 'splices', + 'pendingSplices', + 'queuedBytes' + ]) { + if (!Number.isSafeInteger(counts?.[field]) || counts[field] < 0) { + throw new Error(`${cellId} runtime metrics have invalid ${field}`) + } + } + return counts +} + +function processCounts(config, deps, status, cellId) { + if (status.process) return validatedProcessCounts(status.process, cellId) + const filter = [ + 'resource.type="gce_instance"', + 'jsonPayload.event="orca_relay_runtime_metrics"', + `jsonPayload.cellId="${cellId}"` + ].join(' AND ') + const entries = deps.commandJson([ + 'logging', + 'read', + filter, + '--project', + config.project, + '--freshness=5m', + '--limit=1', + '--order=desc', + '--format=json' + ]) + const entry = entries[0] + if (!entry?.jsonPayload) throw new Error(`${cellId} has no fresh runtime metrics`) + const timestamp = Date.parse(entry.timestamp) + if (!Number.isFinite(timestamp) || timestamp < deps.now() - 90_000) { + throw new Error(`${cellId} runtime metrics are stale`) + } + return validatedProcessCounts(entry.jsonPayload, cellId) +} + +function runtimeConnections(counts, cellId) { + const count = counts?.totalConnections + if (!Number.isSafeInteger(count) || count < 0) { + throw new Error(`${cellId} runtime does not expose totalConnections`) + } + return count +} + +function runtimeIncarnation(status, cellId) { + const incarnation = status.runtime?.cellIncarnation + if (typeof incarnation !== 'string' || incarnation.length === 0) { + throw new Error(`${cellId} has no exact runtime incarnation`) + } + return incarnation +} + +async function pairStatus(config, adminPost, targetCellId, completeReady) { + return await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { + v: 1, + sourceCellId: config.sourceCellId, + targetCellId, + completeReady + }) +} + +async function allPairStatuses(config, adminPost, completeReady) { + const statuses = [] + for (const targetCellId of config.targetCellIds) { + statuses.push({ + targetCellId, + status: await pairStatus(config, adminPost, targetCellId, completeReady) + }) + } + return statuses +} + +function statusTotals(statuses) { + return statuses.reduce( + (totals, { status }) => ({ + inProgress: totals.inProgress + status.inProgress, + targetRegistered: totals.targetRegistered + status.targetRegistered, + registeredSourceActive: totals.registeredSourceActive + status.registeredSourceActive, + registeredCompletable: totals.registeredCompletable + status.registeredCompletable, + registeredTargetInactive: + totals.registeredTargetInactive + status.registeredTargetInactive, + completed: totals.completed + status.completed, + blocked: totals.blocked + status.blocked, + expiredUnregistered: totals.expiredUnregistered + status.expiredUnregistered, + repairableExpiredUnregistered: + totals.repairableExpiredUnregistered + status.repairableExpiredUnregistered, + abortableExpiredUnregistered: + totals.abortableExpiredUnregistered + status.abortableExpiredUnregistered, + blockedExpiredUnregistered: + totals.blockedExpiredUnregistered + status.blockedExpiredUnregistered, + blockedExpiredOnNewerTargetAssignment: + totals.blockedExpiredOnNewerTargetAssignment + + status.blockedExpiredOnNewerTargetAssignment + }), + { + inProgress: 0, + targetRegistered: 0, + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 0, + blocked: 0, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 + } + ) +} + +function hasDurableTargetOwnership(totals) { + return ( + totals.inProgress === totals.targetRegistered && + totals.targetRegistered === + totals.registeredSourceActive + + totals.registeredCompletable + + totals.registeredTargetInactive && + totals.registeredSourceActive === 0 && + totals.expiredUnregistered === 0 && + totals.repairableExpiredUnregistered === 0 && + totals.abortableExpiredUnregistered === 0 && + totals.blockedExpiredUnregistered === 0 && + totals.blockedExpiredOnNewerTargetAssignment === 0 + ) +} + +function boundedUnregisteredMigrations(totals, unobservedConnectionBound) { + const unregistered = totals.inProgress - totals.targetRegistered + return ( + Number.isSafeInteger(unobservedConnectionBound) && + unregistered > 0 && + unregistered <= unobservedConnectionBound && + totals.targetRegistered === + totals.registeredSourceActive + + totals.registeredCompletable + + totals.registeredTargetInactive && + totals.registeredSourceActive === 0 && + totals.blocked === 0 && + totals.expiredUnregistered === 0 && + totals.repairableExpiredUnregistered === 0 && + totals.abortableExpiredUnregistered === 0 && + totals.blockedExpiredUnregistered === 0 && + totals.blockedExpiredOnNewerTargetAssignment === 0 + ) +} + +function isSettledOrOffline(totals) { + return ( + hasDurableTargetOwnership(totals) && + totals.registeredCompletable === 0 && + totals.blocked === totals.registeredTargetInactive + ) +} + +function assertLeaseGate(statuses, minimumLeaseRemainingMs) { + const remaining = statuses + .map(({ status }) => status.oldestRemainingMs) + .filter((value) => value !== null) + if (remaining.length === 0 || Math.min(...remaining) < minimumLeaseRemainingMs) { + throw new Error('oldest migration lease has insufficient time remaining') + } +} + +async function targetRuntime(config, adminPost, target) { + if (config.mode !== 'fence-source') { + const runtime = await adminPost(target.origin, '/v1/admin/runtime-status', { v: 1 }) + const count = runtime.runtime?.totalConnections + if (!Number.isSafeInteger(count) || count < 0) { + throw new Error(`${target.cellId} runtime does not expose totalConnections`) + } + if (count >= targetConnectionCeiling(config, target)) { + throw new Error(`${target.cellId} reached the connection ceiling`) + } + return count + } + const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: target.cellId + }) + const status = result.status + if ( + status?.cellId !== target.cellId || + status.cellUrl !== target.origin || + status.runtime?.cellUrl !== target.origin || + status.runtime?.ready !== true || + status.runtime?.heartbeatFresh !== true + ) { + throw new Error(`${target.cellId} has no fresh matching director runtime snapshot`) + } + const values = [ + status.runtime.observedRequests, + status.connectionCapacity?.observedConnections, + status.connectionCapacity?.enforcedConnectionUnits + ] + if (values.some((value) => !Number.isSafeInteger(value) || value < 0)) { + throw new Error(`${target.cellId} has incomplete director runtime counts`) + } + const count = Math.max(...values) + connectionReservationHeadroom(status, target.cellId) + if (count >= Math.min(config.connectionCeiling, status.connectionCapacity.hardCap)) { + throw new Error(`${target.cellId} reached the connection ceiling`) + } + return count +} + +async function checkPublicCellEndpoint(deps, cell, path) { + const response = await deps.fetch(`${cell.origin}${path}`, { + signal: AbortSignal.timeout(15_000) + }) + const body = await response.json().catch(() => ({})) + if (!response.ok || body.ok !== true) { + throw new Error(`${cell.cellId} ${path} is unavailable`) + } +} + +async function inspectGeneralPromotionTarget(config, deps, adminPost, target) { + await checkPublicCellEndpoint(deps, target, '/health') + await checkPublicCellEndpoint(deps, target, '/ready') + const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: target.cellId + }) + const status = result.status + if ( + status?.cellId !== target.cellId || + status.cellUrl !== target.origin || + status.runtime?.cellUrl !== target.origin || + status.runtime?.ready !== true || + status.runtime?.heartbeatFresh !== true + ) { + throw new Error(`${target.cellId} has no fresh matching director runtime snapshot`) + } + connectionReservationHeadroom(status, target.cellId) + const currentConnections = Math.max( + status.runtime.observedRequests, + status.connectionCapacity.observedConnections, + status.connectionCapacity.enforcedConnectionUnits + ) + if ( + !Number.isSafeInteger(currentConnections) || + currentConnections >= targetConnectionCeiling(config, target) + ) { + throw new Error(`${target.cellId} reached the connection ceiling`) + } +} + +function validateReviewedInstanceTemplate(template, expected, capacityPredecessor) { + const startupScript = (template.properties?.metadata?.items ?? []) + .find((item) => item.key === 'startup-script')?.value + const configuredDigest = startupScript?.match( + /ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'/ + )?.[1] + const configuredImages = [ + ...String(startupScript ?? '').matchAll( + /'(?:[a-z0-9.-]+\/)+[a-z0-9._/-]+@(sha256:[a-f0-9]{64})'/g + ) + ].map((match) => match[1]) + if ( + template.selfLink !== expected.generationIdentity || + configuredDigest !== expected.imageDigest || + !configuredImages.includes(expected.imageDigest) + ) { + throw new Error(`${expected.cellId} instance template does not pin the reviewed image`) + } + const hardCaps = [ + ...String(startupScript ?? '').matchAll( + /^ printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '([0-9]+)'$/gm + ) + ].map((match) => Number(match[1])) + const unobservedBounds = [ + ...String(startupScript ?? '').matchAll( + /^ printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '([0-9]+)'$/gm + ) + ].map((match) => Number(match[1])) + const hardCap = hardCaps[0] + const unobservedBound = unobservedBounds[0] + const exactCapacityPredecessor = + capacityPredecessor !== undefined && + hardCaps.length === 1 && + unobservedBounds.length === 1 && + hardCap === capacityPredecessor.hardCap && + unobservedBound === capacityPredecessor.unobservedBound + if (expected.connectionHardCap === undefined) { + if (!exactCapacityPredecessor && (hardCaps.length !== 0 || unobservedBounds.length !== 0)) { + throw new Error(`${expected.cellId} instance template capacity differs from Terraform`) + } + return exactCapacityPredecessor + ? { + ...expected, + connectionHardCap: hardCap, + connectionUnobservedBound: unobservedBound + } + : expected + } + const isReviewedPredecessor = + exactCapacityPredecessor && expected.connectionHardCap === 1_000 + if ( + hardCaps.length !== 1 || + unobservedBounds.length !== 1 || + !Number.isSafeInteger(hardCap) || + !Number.isSafeInteger(unobservedBound) || + (hardCap !== expected.connectionHardCap && !isReviewedPredecessor) || + unobservedBound !== expected.connectionUnobservedBound + ) { + throw new Error(`${expected.cellId} instance template capacity is outside reviewed rollout`) + } + return { + ...expected, + connectionHardCap: hardCap, + connectionUnobservedBound: unobservedBound + } +} + +async function inspectDirectorObservedCell(config, deps, adminPost, expected) { + const common = ['--project', config.project, '--zone', expected.zone, '--format=json'] + const mig = deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'describe', + expected.migName, + ...common + ]) + const instances = deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'list-instances', + expected.migName, + ...common + ]) + const instanceName = validateMig(mig, instances, expected) + if ( + mig.instanceTemplate !== expected.generationIdentity || + instances[0]?.version?.instanceTemplate !== expected.generationIdentity + ) { + throw new Error(`${expected.cellId} MIG does not serve the reviewed generation`) + } + const instance = deps.commandJson([ + 'compute', + 'instances', + 'describe', + instanceName, + ...common + ]) + validateInstance(instance, expected, config.runtimeServiceAccount) + const templateName = new URL(expected.generationIdentity).pathname.split('/').at(-1) + const template = deps.commandJson([ + 'compute', + 'instance-templates', + 'describe', + templateName, + '--project', + config.project, + '--format=json' + ]) + const deployed = validateReviewedInstanceTemplate( + template, + expected, + config.mode === 'fence-source' && + (expected.connectionHardCap !== undefined || expected.legacyCapacityTopology) + ? { + hardCap: config.connectionCeiling, + unobservedBound: config.unobservedConnectionBound + } + : undefined + ) + const backend = deps.commandJson([ + 'compute', + 'backend-services', + 'describe', + expected.backendName, + '--global', + '--project', + config.project, + '--format=json' + ]) + validateBackend(backend, expected) + await assertCellRoute(config, deps, expected) + await checkPublicCellEndpoint(deps, expected, '/health') + await checkPublicCellEndpoint(deps, expected, '/ready') + const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: expected.cellId + }) + const status = result.status + if ( + status?.cellId !== expected.cellId || + status.cellUrl !== expected.origin || + status.runtime?.cellUrl !== expected.origin || + status.runtime?.ready !== true || + status.runtime?.heartbeatFresh !== true + ) { + throw new Error(`${expected.cellId} has no fresh matching director runtime snapshot`) + } + connectionReservationHeadroom(status, expected.cellId) + assertDeploymentConnectionCapacity( + deployed, + status.connectionCapacity ?? null, + status.connectionCapacity ?? null + ) + const connectionValues = [ + status.runtime.observedRequests, + status.connectionCapacity.observedConnections, + status.connectionCapacity.enforcedConnectionUnits + ] + if (connectionValues.some((value) => !Number.isSafeInteger(value) || value < 0)) { + throw new Error(`${expected.cellId} has incomplete director runtime counts`) + } + const currentConnections = Math.max(...connectionValues) + if (currentConnections >= targetConnectionCeiling(config, deployed)) { + throw new Error(`${expected.cellId} reached the connection ceiling`) + } + return { + ...status, + process: { totalConnections: currentConnections } + } +} + +async function waitForMultiStatus( + config, + deps, + adminPost, + targets, + completeReady, + allowBoundedUnregistered = false, + requireZero = false +) { + const deadline = deps.now() + config.timeoutMs + while (deps.now() < deadline) { + const statuses = await allPairStatuses(config, adminPost, completeReady) + for (const target of targets) await targetRuntime(config, adminPost, target) + const totals = statusTotals(statuses) + deps.emit({ + event: completeReady ? 'multi_migration_completion' : 'multi_migration_registration', + ...totals + }) + const boundedUnregistered = + allowBoundedUnregistered && + boundedUnregisteredMigrations(totals, config.unobservedConnectionBound) + const boundedSettlement = boundedUnregistered && totals.registeredCompletable === 0 + if ( + completeReady + ? (requireZero + ? totals.inProgress === 0 + : isSettledOrOffline(totals) || boundedSettlement) + : totals.inProgress === totals.targetRegistered || boundedUnregistered + ) { + if (boundedUnregistered) { + deps.emit({ + event: completeReady + ? 'multi_migration_bounded_offline_complete' + : 'multi_migration_bounded_offline_registered', + unobservedConnectionBound: config.unobservedConnectionBound, + unregistered: totals.inProgress - totals.targetRegistered, + ...totals + }) + } + return statuses + } + await deps.wait(config.pollIntervalMs) + } + throw new Error( + completeReady + ? 'timed out waiting for multi-target completion' + : 'timed out waiting for multi-target registration' + ) +} + +async function waitForRecoveredSourceZero( + config, + deps, + adminPost, + source, + expectedIncarnation +) { + const deadline = deps.now() + config.timeoutMs + while (deps.now() < deadline) { + const status = await inspectCell(config, deps, adminPost, source) + const counts = processCounts(config, deps, status, source.cellId) + deps.emit({ + event: 'source_recovery_runtime_settlement', + draining: status.draining, + activityLeases: status.activityLeases, + reservedRequests: status.reservedRequests, + controls: counts.controls, + splices: counts.splices, + pendingSplices: counts.pendingSplices + }) + if ( + runtimeIncarnation(status, source.cellId) !== expectedIncarnation + ) { + throw new Error('source incarnation changed during recovery settlement') + } + if ( + status.draining && + status.activityLeases === 0 && + status.reservedRequests === 0 && + counts.controls === 0 && + counts.splices === 0 && + counts.pendingSplices === 0 + ) { + return + } + await deps.wait(config.pollIntervalMs) + } + throw new Error('timed out waiting for recovered source runtime to reach zero') +} + +async function rollbackBeforeDrain(config, deps, adminPost, targets, selectorActive) { + let statuses + try { + statuses = await allPairStatuses(config, adminPost, false) + } catch (error) { + deps.emit({ + event: 'multi_forward_recovery_required', + targetRegistered: null, + reason: 'registration_status_unavailable' + }) + throw new Error('cannot prove zero target registrations; preserving forward recovery', { + cause: error + }) + } + const totals = statusTotals(statuses) + if (totals.targetRegistered > 0) { + deps.emit({ event: 'multi_forward_recovery_required', ...totals }) + return + } + if (selectorActive) { + deps.emit({ event: 'multi_rollback_preserved_selector', ...totals }) + return + } + await setCellState(config, adminPost, config.sourceCellId, true).catch(() => undefined) + for (const target of targets) { + await setCellState(config, adminPost, target.cellId, false).catch(() => undefined) + } + deps.emit({ event: 'multi_rollback_waiting_for_lease_expiry', ...totals }) +} + +async function publishMigrations(config, deps, adminPost, plannedTargets) { + for (const target of plannedTargets) { + let remaining = target.quota + while (remaining > 0) { + const limit = Math.min(config.batchSize, remaining) + let result + try { + result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', { + v: 1, + sourceCellId: config.sourceCellId, + targetCellId: target.cellId, + limit + }) + } catch (error) { + if ( + config.mode !== 'recover-forward' || + !(error instanceof Error) || + error.message !== + '/v1/admin/evacuate-cell failed: relay_connection_headroom_exhausted' + ) { + throw error + } + deps.emit({ + event: 'multi_recovery_target_headroom_paused', + targetCellId: target.cellId, + remaining + }) + break + } + if ( + !Number.isSafeInteger(result.started) || + result.started < 0 || + result.started > limit + ) { + throw new Error(`${target.cellId} migration quota could not be filled deterministically`) + } + if (result.started === 0 && config.mode === 'recover-forward') { + deps.emit({ + event: 'multi_recovery_quota_depleted', + targetCellId: target.cellId, + remaining + }) + break + } + if (result.started === 0) { + throw new Error(`${target.cellId} migration quota could not be filled deterministically`) + } + remaining -= result.started + deps.emit({ + event: 'multi_migration_batch', + targetCellId: target.cellId, + started: result.started, + remaining + }) + } + } +} + +function sourceMig(config, deps, source) { + const common = ['--project', config.project, '--zone', source.zone, '--format=json'] + return { + mig: deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'describe', + source.migName, + ...common + ]), + instances: deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'list-instances', + source.migName, + ...common + ]) + } +} + +function validateFencedMig(mig, source) { + const policy = mig.updatePolicy ?? {} + if ( + Number(mig.targetSize) !== 0 || + mig.instanceTemplate !== source.generationIdentity || + policy.replacementMethod !== 'RECREATE' || + Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 || + Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1 + ) { + throw new Error(`${source.cellId} MIG fence topology is unsafe`) + } +} + +function canonicalBackendServiceId(value) { + if (typeof value !== 'string') return null + const prefix = 'https://www.googleapis.com/compute/v1/' + const resource = value.startsWith(prefix) ? value.slice(prefix.length) : value + return /^projects\/[a-z][a-z0-9-]{4,29}\/global\/backendServices\/[A-Za-z0-9_-]{1,63}$/.test( + resource + ) + ? resource + : null +} + +export function sameBackendServiceResource(left, right) { + if (left === right) return true + const leftId = canonicalBackendServiceId(left) + return leftId !== null && leftId === canonicalBackendServiceId(right) +} + +function validateRetainedRoute(urlMap, source) { + const hostname = new URL(source.origin).hostname + const hostRule = (urlMap.hostRules ?? []).find((rule) => + (rule.hosts ?? []).includes(hostname) + ) + const matcher = (urlMap.pathMatchers ?? []).find( + (candidate) => candidate.name === hostRule?.pathMatcher + ) + if ( + !source.urlMapName || + urlMap.name !== source.urlMapName || + !sameBackendServiceResource(matcher?.defaultService, source.backendId) || + (matcher.pathRules?.length ?? 0) !== 0 || + (matcher.routeRules?.length ?? 0) !== 0 || + matcher.defaultRouteAction !== undefined || + matcher.defaultUrlRedirect !== undefined || + matcher.headerAction !== undefined + ) { + throw new Error(`${source.cellId} retained route topology mismatch`) + } +} + +async function assertCellRoute(config, deps, cell) { + const urlMap = deps.commandJson([ + 'compute', + 'url-maps', + 'describe', + cell.urlMapName, + '--global', + '--project', + config.project, + '--format=json' + ]) + validateRetainedRoute(urlMap, cell) + const proxy = deps.commandJson([ + 'compute', + 'target-https-proxies', + 'describe', + cell.urlMapName, + '--global', + '--project', + config.project, + '--format=json' + ]) + const forwardingRule = deps.commandJson([ + 'compute', + 'forwarding-rules', + 'describe', + cell.urlMapName, + '--global', + '--project', + config.project, + '--format=json' + ]) + const address = deps.commandJson([ + 'compute', + 'addresses', + 'describe', + cell.urlMapName, + '--global', + '--project', + config.project, + '--format=json' + ]) + const resolved = await deps.resolve4(new URL(cell.origin).hostname) + if ( + proxy.name !== cell.urlMapName || + proxy.urlMap !== urlMap.selfLink || + forwardingRule.name !== cell.urlMapName || + forwardingRule.target !== proxy.selfLink || + forwardingRule.IPAddress !== address.address || + forwardingRule.portRange !== '443-443' || + forwardingRule.loadBalancingScheme !== 'EXTERNAL_MANAGED' || + !Array.isArray(resolved) || + resolved.length === 0 || + resolved.some((value) => value !== address.address) + ) { + throw new Error(`${cell.cellId} live frontend topology mismatch`) + } +} + +async function inspectFencedSource(config, deps, adminPost, source, mig) { + validateFencedMig(mig, source) + const backend = deps.commandJson([ + 'compute', + 'backend-services', + 'describe', + source.backendName, + '--global', + '--project', + config.project, + '--format=json' + ]) + validateBackend(backend, source) + await assertCellRoute(config, deps, source) + const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: source.cellId + }) + const status = result.status + if ( + status?.cellUrl !== source.origin || + (status.runtime !== null && status.runtime?.cellUrl !== source.origin) + ) { + throw new Error(`${source.cellId} fenced runtime does not match Terraform topology`) + } + return { ...status, draining: true, process: null } +} + +async function inspectFenceCandidate(config, deps, adminPost, cell, mig) { + const targetSize = Number(mig.targetSize) + const policy = mig.updatePolicy ?? {} + if ( + ![0, 1].includes(targetSize) || + policy.replacementMethod !== 'RECREATE' || + Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 || + Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1 + ) { + throw new Error(`${cell.cellId} MIG fence topology is unsafe`) + } + const backend = deps.commandJson([ + 'compute', + 'backend-services', + 'describe', + cell.backendName, + '--global', + '--project', + config.project, + '--format=json' + ]) + validateBackend(backend, cell) + const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: cell.cellId + }) + if ( + result.status?.cellUrl !== cell.origin || + result.status.runtime?.cellUrl !== cell.origin + ) { + throw new Error(`${cell.cellId} retained topology does not match Terraform`) + } + return result.status +} + +const CAPACITY_SNAPSHOT_ATTEMPTS = 3 +const CAPACITY_SNAPSHOT_RETRY_MS = 250 +const RECOVERY_CATCH_UP_PASSES = 5 +const RECOVERY_TARGET_OWNERSHIP_TIMEOUT_MS = 2 * 60 * 1_000 + +function conservativeRecoveryCapacity(rounds, targets, requireZeroProof) { + const capacities = rounds.flat() + for (const capacity of capacities) { + if ( + !Number.isSafeInteger(capacity.sourceAssignments) || + capacity.sourceAssignments < 0 || + !Number.isSafeInteger(capacity.requiredTargetUnits) || + capacity.requiredTargetUnits < capacity.sourceAssignments || + !Number.isSafeInteger(capacity.availableTargetUnits) || + capacity.availableTargetUnits < 0 + ) { + throw new Error('target capacity snapshot is internally inconsistent') + } + } + const observedSourceAssignments = capacities.map( + (capacity) => capacity.sourceAssignments + ) + const sourceAssignments = requireZeroProof + ? Math.max(...observedSourceAssignments) + : Math.min(...observedSourceAssignments) + const requiredTargetUnits = + sourceAssignments === 0 + ? 0 + : requireZeroProof + ? Math.max(...capacities.map((capacity) => capacity.requiredTargetUnits)) + : Math.max( + ...capacities.map((capacity) => + Math.ceil( + sourceAssignments * + (capacity.requiredTargetUnits / capacity.sourceAssignments) + ) + ) + ) + return { + sourceAssignments, + requiredTargetUnits, + availableTargetUnits: new Map(targets.map((target, index) => [ + target.cellId, + Math.min(...rounds.map((round) => round[index].availableTargetUnits)) + ])) + } +} + +async function readTargetCapacitySnapshot( + config, + deps, + adminPost, + source, + targets, + requireRecoveryZeroProof = false +) { + for (let attempt = 0; attempt < CAPACITY_SNAPSHOT_ATTEMPTS; attempt++) { + const rounds = [] + for (let round = 0; round < 2; round++) { + const capacities = [] + for (const target of targets) { + capacities.push(await adminPost( + config.directorOrigin, + '/v1/admin/evacuation-capacity', + { v: 1, sourceCellId: source.cellId, targetCellId: target.cellId } + )) + } + rounds.push(capacities) + } + if (config.mode === 'recover-forward') { + return conservativeRecoveryCapacity(rounds, targets, requireRecoveryZeroProof) + } + const capacities = rounds.flat() + const baseline = capacities[0] + if (capacities.every((capacity) => + capacity.sourceAssignments === baseline.sourceAssignments && + capacity.requiredTargetUnits === baseline.requiredTargetUnits + )) { + return { + sourceAssignments: baseline.sourceAssignments, + requiredTargetUnits: baseline.requiredTargetUnits, + availableTargetUnits: new Map(targets.map((target, index) => [ + target.cellId, + Math.min(...rounds.map((round) => round[index].availableTargetUnits)) + ])) + } + } + if (attempt + 1 < CAPACITY_SNAPSHOT_ATTEMPTS) { + await deps.wait(CAPACITY_SNAPSHOT_RETRY_MS) + } + } + throw new Error('target capacity snapshots disagree') +} + +async function preflight( + config, + deps, + adminPost, + source, + targets, + sourceFence = null, + selector = null, + coveredSourceConnections = null +) { + const sourceStatus = sourceFence + ? await inspectFencedSource(config, deps, adminPost, source, sourceFence.mig) + : config.mode === 'fence-source' + ? { + ...await inspectDirectorObservedCell(config, deps, adminPost, source), + process: null + } + : await inspectCell(config, deps, adminPost, source) + const sourceProcess = sourceFence + ? null + : processCounts(config, deps, sourceStatus, source.cellId) + const targetStatuses = [] + for (const target of targets) { + const status = config.mode === 'fence-source' + ? { + ...await inspectDirectorObservedCell(config, deps, adminPost, target), + process: null + } + : await inspectCell(config, deps, adminPost, target) + const targetProcess = processCounts(config, deps, status, target.cellId) + const targetAdmission = selector + ? selectorCellState(selector, target.cellId) + : status.enabled + ? 'general' + : 'existing-only' + if ( + ['preflight', 'execute'].includes(config.mode) && + (selector ? targetAdmission === 'general' : status.enabled) + ) { + throw new Error(`${target.cellId} must not start in general admission`) + } + targetStatuses.push({ + ...target, + status, + process: targetProcess, + currentConnections: runtimeConnections(targetProcess, target.cellId), + availableConnectionReservations: connectionReservationHeadroom( + status, + target.cellId + ), + connectionCeiling: Math.min( + config.connectionCeiling, + status.connectionCapacity.hardCap + ) + }) + } + if (config.mode === 'audit' || config.mode === 'recover-forward') { + const statuses = await allPairStatuses(config, adminPost, false) + deps.emit({ + event: config.mode === 'audit' ? 'multi_target_audit' : 'multi_forward_recovery_preflight', + ...statusTotals(statuses) + }) + if (config.mode === 'audit') { + return { + sourceProcess, + sourceStatus, + plannedTargets: targetStatuses, + sourceAlreadyFenced: Boolean(sourceFence) + } + } + } + const capacity = await readTargetCapacitySnapshot( + config, + deps, + adminPost, + source, + targets + ) + const { sourceAssignments, requiredTargetUnits } = capacity + const observedSourceConnections = sourceFence + ? 0 + : runtimeConnections(sourceProcess, source.cellId) + const sourceConnections = + coveredSourceConnections === null + ? observedSourceConnections + : sourceAssignments + + Math.max(0, observedSourceConnections - coveredSourceConnections) + for (const target of targetStatuses) { + target.availableTargetUnits = capacity.availableTargetUnits.get(target.cellId) + } + deps.emit({ + event: 'multi_target_capacity_snapshot', + sourceConnections, + observedSourceConnections, + sourceAssignments, + requiredTargetUnits, + targets: targetStatuses.map((target) => ({ + cellId: target.cellId, + currentConnections: target.currentConnections, + availableConnectionReservations: target.availableConnectionReservations, + availableTargetUnits: target.availableTargetUnits + })) + }) + if ( + config.mode === 'execute' && + (selector + ? selectorCellState(selector, source.cellId) !== 'existing-only' + : !sourceStatus.enabled) + ) { + throw new Error(selector ? 'source cell is not existing-only' : 'source cell is not enabled') + } + const plannedTargets = allocateTargetQuotas({ + sourceAssignments, + sourceConnections, + requiredTargetUnits, + targets: targetStatuses, + connectionCeiling: config.connectionCeiling + }) + deps.emit({ + event: 'multi_target_preflight', + source: aggregateCellStatus(sourceStatus), + sourceConnections, + observedSourceConnections, + sourceAssignments, + targets: plannedTargets.map((target) => ({ + cellId: target.cellId, + quota: target.quota, + currentConnections: target.currentConnections, + projectedConnections: target.projectedConnections, + projectedUnits: target.projectedUnits + })) + }) + return { sourceProcess, sourceStatus, plannedTargets, sourceAlreadyFenced: Boolean(sourceFence) } +} + +async function assertRecoveryPreDrain( + config, + deps, + adminPost, + source, + targets, + selectorPost, + expectedSelector +) { + const statuses = await allPairStatuses(config, adminPost, false) + const totals = statusTotals(statuses) + const capacity = await readTargetCapacitySnapshot( + config, + deps, + adminPost, + source, + targets, + true + ) + if (capacity.sourceAssignments !== 0 || capacity.requiredTargetUnits !== 0) { + deps.emit({ + event: 'multi_forward_recovery_catch_up', + sourceAssignments: capacity.sourceAssignments, + requiredTargetUnits: capacity.requiredTargetUnits + }) + return false + } + for (const target of targets) await targetRuntime(config, adminPost, target) + if (expectedSelector) { + const current = await inspectAdmissionSelector(selectorPost) + if ( + current.selector.generation !== expectedSelector.generation || + JSON.stringify(current.selector.membership) !== + JSON.stringify(expectedSelector.membership) + ) { + throw new Error('admission selector changed before recovery drain') + } + } + deps.emit({ + event: 'multi_forward_recovery_ready_to_drain', + ...totals + }) + return true +} + +async function waitForRecoveryTargetOwnership(config, deps, adminPost, targets) { + const deadline = + deps.now() + Math.min(config.timeoutMs, RECOVERY_TARGET_OWNERSHIP_TIMEOUT_MS) + while (deps.now() < deadline) { + const statuses = await allPairStatuses(config, adminPost, false) + for (const target of targets) await targetRuntime(config, adminPost, target) + const totals = statusTotals(statuses) + deps.emit({ event: 'multi_recovery_target_ownership', ...totals }) + assertLeaseGate(statuses, config.minimumLeaseRemainingMs) + if (hasDurableTargetOwnership(totals)) return + if (boundedUnregisteredMigrations(totals, config.unobservedConnectionBound)) { + const unregistered = totals.inProgress - totals.targetRegistered + deps.emit({ + event: + totals.targetRegistered === 0 + ? 'multi_recovery_bounded_unregistered' + : 'multi_recovery_bounded_mixed_registration', + unobservedConnectionBound: config.unobservedConnectionBound, + unregistered, + ...totals + }) + return + } + await deps.wait(config.pollIntervalMs) + } + throw new Error('timed out waiting for recovery target ownership') +} + +async function runEvacuation( + config, + deps, + adminPost, + token, + source, + targets, + plannedTargets, + sourceStatus, + selectorActive, + selectorPost, + expectedSelector +) { + let drainAttempted = false + try { + if (!selectorActive) { + await setCellState(config, adminPost, source.cellId, false) + for (const target of targets) await setCellState(config, adminPost, target.cellId, true) + } + await publishMigrations(config, deps, adminPost, plannedTargets) + const statuses = await allPairStatuses(config, adminPost, false) + assertLeaseGate(statuses, config.minimumLeaseRemainingMs) + for (const target of targets) await targetRuntime(config, adminPost, target) + if (selectorActive) { + const current = await inspectAdmissionSelector(selectorPost) + if ( + current.selector.generation !== expectedSelector.generation || + JSON.stringify(current.selector.membership) !== + JSON.stringify(expectedSelector.membership) + ) { + throw new Error('admission selector changed before drain') + } + } + const attemptId = randomUUID() + const traceValue = randomUUID() + const prepared = await adminPost( + config.directorOrigin, + '/v1/admin/drain-attempt-prepare', + { + v: 1, + attemptId, + cellId: source.cellId, + cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId), + traceValue, + graceMs: 120_000, + confirmation: 'PREPARE_LEGACY_DRAIN' + } + ) + if (prepared.state !== 'prepared') { + throw new Error('planned drain already recorded; use recover-forward') + } + drainAttempted = true + const sending = await adminPost( + config.directorOrigin, + '/v1/admin/drain-attempt-send', + { + v: 1, + attemptId, + cellId: source.cellId, + cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId) + } + ) + if ( + sending.attempt?.state !== 'send-may-have-started' || + sending.attempt.shouldSend !== true || + !Number.isSafeInteger(sending.attempt.sendPermitExpiresAt) || + deps.now() >= sending.attempt.sendPermitExpiresAt + ) { + throw new Error('drain send permit unavailable') + } + const receipt = await drainSource(config, deps, token, source, 120_000, traceValue) + await adminPost(config.directorOrigin, '/v1/admin/drain-attempt-receipt', { + v: 1, + attemptId, + cellId: source.cellId, + cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId), + traceValue, + ...receipt + }) + deps.emit({ event: 'source_drain_accepted', sourceCellId: source.cellId }) + await waitForMultiStatus(config, deps, adminPost, targets, false) + await waitForMultiStatus(config, deps, adminPost, targets, true) + deps.emit({ event: 'multi_target_complete', sourceCellId: source.cellId }) + } catch (error) { + if (drainAttempted) { + const statuses = await allPairStatuses(config, adminPost, false).catch(() => []) + deps.emit({ event: 'multi_forward_recovery_required', ...statusTotals(statuses) }) + } else { + await rollbackBeforeDrain(config, deps, adminPost, targets, selectorActive) + } + throw error + } +} + +async function waitForFence(config, deps, adminPost, source) { + const deadline = deps.now() + config.timeoutMs + while (deps.now() < deadline) { + const mig = deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'describe', + source.migName, + '--project', + config.project, + '--zone', + source.zone, + '--format=json' + ]) + const instances = deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'list-instances', + source.migName, + '--project', + config.project, + '--zone', + source.zone, + '--format=json' + ]) + const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: source.cellId + }) + if ( + Number(mig.targetSize) === 0 && + instances.length === 0 && + status.status?.cellUrl === source.origin && + status.status.enabled === false && + !status.status.runtime?.heartbeatFresh + ) { + const incarnation = status.status.runtime?.cellIncarnation + if (typeof incarnation !== 'string' || incarnation.length === 0) { + throw new Error('fenced source has no exact runtime incarnation') + } + return incarnation + } + await deps.wait(config.pollIntervalMs) + } + throw new Error('timed out waiting for durable source fence') +} + +function terraformFenceConfig(config, cell, cellIncarnation) { + return { + project: config.project, + environment: config.environment, + terraformDir: config.terraformDir, + varFile: config.terraformVarFile, + lockTimeout: '5m', + fenceCommit: config.fenceCommit, + cellIncarnation, + cell + } +} + +function fenceAttemptBody(attempt) { + return { + v: 1, + attemptId: attempt.attemptId, + environment: attempt.environment, + cellId: attempt.cellId, + cellIncarnation: attempt.cellIncarnation, + migName: attempt.migName, + instanceGroup: attempt.instanceGroup, + generationIdentity: attempt.generationIdentity, + fenceCommit: attempt.fenceCommit, + planSha256: attempt.planSha256, + planObjectName: attempt.planObjectName, + planObjectGeneration: attempt.planObjectGeneration, + varFileSha256: attempt.varFileSha256, + terraformStateLineage: attempt.terraformStateLineage, + terraformStateSerial: attempt.terraformStateSerial, + terraformStateObjectGeneration: attempt.terraformStateObjectGeneration, + terraformStateObjectSha256: attempt.terraformStateObjectSha256, + requestReason: attempt.requestReason, + ...(attempt.gceOperation ? { gceOperation: attempt.gceOperation } : {}) + } +} + +async function runTerraformManagedFence( + config, + deps, + adminPost, + cell, + cellIncarnation, + alreadyFenced, + preApplyGuard, + postApplyGuard +) { + const fenceConfig = terraformFenceConfig(config, cell, cellIncarnation) + const inspectProgress = async (_expected, attempt) => + await inspectTerraformFenceProgress( + fenceConfig, + { + terraform: deps.terraform, + gcloudJson: deps.commandJson + }, + attempt + ) + const attest = async (attempt) => { + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attest', { + ...fenceAttemptBody(attempt), + confirmation: 'ATTEST_TERRAFORM_FENCED_CELL' + }) + } + const attemptResult = await adminPost( + config.directorOrigin, + '/v1/admin/cell-fence-attempt-status', + { v: 1, cellId: cell.cellId } + ) + let existingAttempt = attemptResult.attempt + const planStore = { + uploadPlan: async (planPath, attempt) => + await uploadTerraformFencePlan( + fenceConfig, + { command: deps.command, commandJson: deps.commandJson }, + planPath, + attempt + ), + downloadPlan: async (attempt, planPath) => + await downloadTerraformFencePlan( + fenceConfig, + { command: deps.command }, + attempt, + planPath + ), + deletePlan: async (attempt) => + await deleteTerraformFencePlan( + fenceConfig, + { commandResult: deps.commandResult }, + attempt + ), + stateObjectBinding: async (statePath) => + await readTerraformStateObjectBinding( + fenceConfig, + { command: deps.command, commandJson: deps.commandJson }, + statePath + ) + } + if ( + existingAttempt && + existingAttempt.fenceCommit !== config.fenceCommit && + config.completedFenceRecovery + ) { + await deps.terraformFenceRecoverCompleted( + fenceConfig, + { + terraform: deps.terraform, + assertCommittedFenceSet: async () => + assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), + loadAttempt: async () => existingAttempt, + resolvePlan: async (attempt) => + await resolveTerraformFencePlanGeneration( + fenceConfig, + { commandResult: deps.commandResult }, + attempt + ), + stateObjectBinding: planStore.stateObjectBinding, + downloadPlan: planStore.downloadPlan, + deletePlan: planStore.deletePlan, + inspectCompletedProgress: async (_expected, attempt, recovery) => + await inspectCompletedTerraformFenceProgress( + fenceConfig, + { + terraform: deps.terraform, + gcloudJson: deps.commandJson + }, + attempt, + recovery + ), + markOperation: async (attempt, invocation) => + await adminPost( + config.directorOrigin, + '/v1/admin/cell-fence-attempt-operation', + { + ...fenceAttemptBody(attempt), + invocationId: invocation.invocationId, + invocationRequestReason: invocation.requestReason, + confirmation: 'RECORD_TERRAFORM_CELL_FENCE_OPERATION' + } + ), + assertZeroDiff: async () => + assertTerraformFenceZeroDiff(fenceConfig, { + terraform: deps.terraform + }), + postApplyGuard, + attest, + emit: deps.emit + }, + config.completedFenceRecovery + ) + return + } + if ( + existingAttempt && + !existingAttempt.abortedAt && + !existingAttempt.completedAt && + existingAttempt.fenceCommit !== config.fenceCommit + ) { + await deps.terraformFenceSupersede(fenceConfig, { + assertCommittedFenceSet: async () => + assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), + loadAttempt: async () => existingAttempt, + resolvePlan: async (attempt) => + await resolveTerraformFencePlanGeneration( + fenceConfig, + { commandResult: deps.commandResult }, + attempt + ), + inspectProgress, + abortAttempt: async (attempt) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-abort', { + ...fenceAttemptBody(attempt), + confirmation: 'ABORT_UNSTARTED_TERRAFORM_CELL_FENCE' + }), + emit: deps.emit + }) + existingAttempt = null + } + if ( + existingAttempt && + !existingAttempt.abortedAt && + (alreadyFenced || !existingAttempt.completedAt) + ) { + await deps.terraformFenceResume(fenceConfig, { + terraform: deps.terraform, + assertCommittedFenceSet: async () => + assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), + loadAttempt: async () => existingAttempt, + resolvePlan: async (attempt) => + await resolveTerraformFencePlanGeneration( + fenceConfig, + { commandResult: deps.commandResult }, + attempt + ), + bindPlan: async (attempt) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-plan', { + ...fenceAttemptBody(attempt), + confirmation: 'BIND_TERRAFORM_CELL_FENCE_PLAN' + }), + inspectProgress, + assertZeroDiff: async () => + assertTerraformFenceZeroDiff(fenceConfig, { terraform: deps.terraform }), + assertStateFenced: async () => + assertTerraformFenceStateFenced(fenceConfig, { terraform: deps.terraform }), + preApplyGuard, + postApplyGuard, + markOperation: async (attempt, invocation) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-operation', { + ...fenceAttemptBody(attempt), + invocationId: invocation.invocationId, + invocationRequestReason: invocation.requestReason, + confirmation: 'RECORD_TERRAFORM_CELL_FENCE_OPERATION' + }), + markApplyStarted: async (attempt, invocation) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-start', { + ...fenceAttemptBody(attempt), + invocationId: invocation.invocationId, + invocationRequestReason: invocation.requestReason, + confirmation: 'START_TERRAFORM_CELL_FENCE' + }), + attest, + ...planStore, + emit: deps.emit + }) + return + } + if (alreadyFenced) { + await deps.terraformFenceAdopt(fenceConfig, { + loadAttempt: async () => + ( + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-status', { + v: 1, + cellId: cell.cellId + }) + ).attempt, + assertCommittedFenceSet: async () => + assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), + assertStateFenced: async () => + assertTerraformFenceStateFenced(fenceConfig, { terraform: deps.terraform }), + preApplyGuard, + postApplyGuard, + attest: async (incarnation) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-adopt-legacy', { + v: 1, + cellId: cell.cellId, + cellIncarnation: incarnation, + confirmation: 'ADOPT_LEGACY_TERRAFORM_CELL_FENCE' + }), + commitAdoption: async (incarnation) => + await adminPost( + config.directorOrigin, + '/v1/admin/cell-fence-commit-legacy-adoption', + { + v: 1, + cellId: cell.cellId, + cellIncarnation: incarnation, + confirmation: 'COMMIT_LEGACY_TERRAFORM_CELL_FENCE_ADOPTION' + } + ), + emit: deps.emit + }) + return + } + if (existingAttempt && !existingAttempt.abortedAt && !existingAttempt.completedAt) { + throw new Error('prepared Terraform fence attempt must be aborted before replacement') + } + await deps.terraformFenceApply(fenceConfig, { + terraform: deps.terraform, + assertCommittedFenceSet: async () => + assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), + inspectProgress, + assertZeroDiff: async () => + assertTerraformFenceZeroDiff(fenceConfig, { terraform: deps.terraform }), + preApplyGuard, + postApplyGuard, + ...planStore, + prepareAttempt: async (attempt) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-prepare', { + ...fenceAttemptBody(attempt), + confirmation: 'PREPARE_TERRAFORM_CELL_FENCE' + }), + bindPlan: async (attempt) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-plan', { + ...fenceAttemptBody(attempt), + confirmation: 'BIND_TERRAFORM_CELL_FENCE_PLAN' + }), + markApplyStarted: async (attempt, invocation) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-start', { + ...fenceAttemptBody(attempt), + invocationId: invocation.invocationId, + invocationRequestReason: invocation.requestReason, + confirmation: 'START_TERRAFORM_CELL_FENCE' + }), + markOperation: async (attempt, invocation) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-operation', { + ...fenceAttemptBody(attempt), + invocationId: invocation.invocationId, + invocationRequestReason: invocation.requestReason, + confirmation: 'RECORD_TERRAFORM_CELL_FENCE_OPERATION' + }), + attest, + emit: deps.emit + }) +} + +async function abortTerraformManagedFence(config, deps, adminPost, cell) { + const result = await adminPost( + config.directorOrigin, + '/v1/admin/cell-fence-attempt-status', + { v: 1, cellId: cell.cellId } + ) + const attempt = result.attempt + const fenceConfig = terraformFenceConfig(config, cell, attempt?.cellIncarnation) + await deps.terraformFenceAbort(fenceConfig, { + terraform: deps.terraform, + assertCommittedFenceSet: async () => + assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), + loadAttempt: async () => attempt, + resolvePlan: async (value) => + await resolveTerraformFencePlanGeneration( + fenceConfig, + { commandResult: deps.commandResult }, + value + ), + bindPlan: async (value) => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-plan', { + ...fenceAttemptBody(value), + confirmation: 'BIND_TERRAFORM_CELL_FENCE_PLAN' + }), + inspectProgress: async () => + await inspectTerraformFenceProgress( + fenceConfig, + { + terraform: deps.terraform, + gcloudJson: deps.commandJson + }, + attempt + ), + abortAttempt: async () => + await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-abort', { + ...fenceAttemptBody(attempt), + confirmation: 'ABORT_UNSTARTED_TERRAFORM_CELL_FENCE' + }), + deletePlan: async (value) => + await deleteTerraformFencePlan( + fenceConfig, + { commandResult: deps.commandResult }, + value + ), + emit: deps.emit + }) +} + +async function fenceSource( + config, + deps, + adminPost, + source, + targets, + sourceProcess, + sourceStatus, + sourceAlreadyFenced +) { + if (sourceStatus.enabled) throw new Error('source fencing requires disabled admission') + const counts = sourceProcess + if ( + (!sourceAlreadyFenced && + (!counts || + sourceStatus.runtime?.observedRequests !== 0 || + counts.controls !== 0 || + counts.splices !== 0 || + counts.pendingSplices !== 0)) || + sourceStatus.activityLeases !== 0 || + sourceStatus.reservedRequests !== 0 + ) { + throw new Error('source fencing requires zero source-owned work') + } + const statuses = await allPairStatuses(config, adminPost, false) + const totals = statusTotals(statuses) + if ( + totals.inProgress !== sourceStatus.outgoingMigrations || + !hasDurableTargetOwnership(totals) + ) { + throw new Error('source fencing requires full migration coverage and durable target ownership') + } + for (const target of targets) await targetRuntime(config, adminPost, target) + const cellIncarnation = runtimeIncarnation(sourceStatus, source.cellId) + const postApplyGuard = async (expectedIncarnation) => { + const actualIncarnation = await waitForFence(config, deps, adminPost, source) + if (actualIncarnation !== expectedIncarnation) { + throw new Error('fenced source incarnation changed') + } + const finalSourceMig = sourceMig(config, deps, source) + if (finalSourceMig.instances.length !== 0) { + throw new Error('fenced source still has an instance') + } + await inspectFencedSource(config, deps, adminPost, source, finalSourceMig.mig) + } + await runTerraformManagedFence( + config, + deps, + adminPost, + source, + cellIncarnation, + sourceAlreadyFenced, + async () => { + const latestStatus = sourceAlreadyFenced + ? await inspectFencedSource( + config, + deps, + adminPost, + source, + sourceMig(config, deps, source).mig + ) + : { + ...await inspectDirectorObservedCell(config, deps, adminPost, source), + process: null + } + const latestCounts = sourceAlreadyFenced + ? null + : processCounts(config, deps, latestStatus, source.cellId) + if ( + latestStatus.enabled || + runtimeIncarnation(latestStatus, source.cellId) !== cellIncarnation || + (!sourceAlreadyFenced && + (latestStatus.runtime?.observedRequests !== 0 || + latestCounts.controls !== 0 || + latestCounts.splices !== 0 || + latestCounts.pendingSplices !== 0)) || + latestStatus.activityLeases !== 0 || + latestStatus.reservedRequests !== 0 + ) { + throw new Error('source fencing guards changed before Terraform apply') + } + const latestStatuses = await allPairStatuses(config, adminPost, false) + const latestTotals = statusTotals(latestStatuses) + if ( + latestTotals.inProgress !== latestStatus.outgoingMigrations || + !hasDurableTargetOwnership(latestTotals) + ) { + throw new Error('source migration coverage or guards changed before Terraform apply') + } + for (const target of targets) { + await inspectDirectorObservedCell(config, deps, adminPost, target) + } + }, + postApplyGuard + ) + await waitForMultiStatus(config, deps, adminPost, targets, true, false, true) + deps.emit({ event: 'source_fenced', sourceCellId: source.cellId, targetSize: 0 }) +} + +async function runTargetSupersession( + config, + deps, + adminPost, + source, + targets, + selector +) { + const failed = targets.find((target) => target.cellId === config.failedTargetCellId) + const replacement = targets.find( + (target) => target.cellId === config.replacementTargetCellId + ) + if (!failed || !replacement) throw new Error('supersession topology is incomplete') + const sourceStatus = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { + v: 1, + cellId: source.cellId + }) + if ( + sourceStatus.status?.cellUrl !== source.origin || + (selector + ? selectorCellState(selector, source.cellId) !== 'existing-only' + : sourceStatus.status.enabled) + ) { + throw new Error('supersession requires retained disabled source topology') + } + const failedMig = sourceMig(config, deps, failed) + const failedStatus = await inspectFenceCandidate( + config, + deps, + adminPost, + failed, + failedMig.mig + ) + if ( + selector + ? selectorCellState(selector, failed.cellId) !== 'existing-only' + : failedStatus.enabled + ) { + throw new Error( + selector + ? 'failed target admission must be existing-only' + : 'failed target admission must be disabled' + ) + } + const replacementStatus = await inspectDirectorObservedCell( + config, + deps, + adminPost, + replacement + ) + const migrationStatus = await pairStatus(config, adminPost, failed.cellId, false) + if ( + !Number.isSafeInteger(migrationStatus.targetRegistered) || + migrationStatus.targetRegistered < 1 + ) { + throw new Error('failed target has no registered migrations to supersede') + } + const failedConnections = failedStatus.runtime?.observedRequests + if (!Number.isSafeInteger(failedConnections) || failedConnections < 0) { + throw new Error('failed target has no exact runtime connection snapshot') + } + const replacementConnections = runtimeConnections( + replacementStatus.process, + replacement.cellId + ) + const projectedConnections = + replacementConnections + + Math.max(failedConnections, migrationStatus.targetRegistered) + if (projectedConnections >= targetConnectionCeiling(config, replacement)) { + throw new Error('replacement target lacks conservative connection headroom') + } + const cellIncarnation = runtimeIncarnation(failedStatus, failed.cellId) + await runTerraformManagedFence( + config, + deps, + adminPost, + failed, + cellIncarnation, + Number(failedMig.mig.targetSize) === 0, + async () => { + const latestMig = sourceMig(config, deps, failed) + const latestStatus = await inspectFenceCandidate( + config, + deps, + adminPost, + failed, + latestMig.mig + ) + if ( + selector + ? selectorCellState(selector, failed.cellId) !== 'existing-only' + : latestStatus.enabled + ) { + throw new Error('failed target admission changed before apply') + } + if (runtimeIncarnation(latestStatus, failed.cellId) !== cellIncarnation) { + throw new Error('failed target incarnation changed before apply') + } + const latestMigration = await pairStatus(config, adminPost, failed.cellId, false) + if (latestMigration.targetRegistered < 1) { + throw new Error('failed target migrations changed before apply') + } + await inspectDirectorObservedCell(config, deps, adminPost, replacement) + }, + async (expectedIncarnation) => { + const actualIncarnation = await waitForFence(config, deps, adminPost, failed) + if (actualIncarnation !== expectedIncarnation) { + throw new Error('failed target incarnation changed') + } + const finalFailedMig = sourceMig(config, deps, failed) + if (finalFailedMig.instances.length !== 0) { + throw new Error('failed target fence still has an instance') + } + await inspectFencedSource(config, deps, adminPost, failed, finalFailedMig.mig) + } + ) + if ( + selector && + selectorCellState(selector, replacement.cellId) !== 'migration-only' + ) { + throw new Error('replacement target admission must be migration-only') + } + if (!selector && !replacementStatus.enabled) { + await setCellState(config, adminPost, replacement.cellId, true) + } + let superseded = 0 + while (true) { + const result = await adminPost( + config.directorOrigin, + '/v1/admin/migration-supersede-cell', + { + v: 1, + sourceCellId: source.cellId, + currentTargetCellId: failed.cellId, + replacementTargetCellId: replacement.cellId, + limit: config.batchSize, + confirmation: 'SUPERSEDE_REGISTERED_CELL_MIGRATIONS' + } + ) + if ( + !Number.isSafeInteger(result.superseded) || + result.superseded < 0 || + result.superseded > config.batchSize + ) { + throw new Error('invalid registered supersession result') + } + superseded += result.superseded + if (result.superseded === 0) break + } + const remaining = await pairStatus(config, adminPost, failed.cellId, false) + if (remaining.targetRegistered !== 0) { + throw new Error('registered target supersession did not reconcile') + } + if (superseded !== migrationStatus.targetRegistered) { + throw new Error('registered target supersession count changed') + } + deps.emit({ + event: 'registered_target_superseded', + sourceCellId: source.cellId, + failedTargetCellId: failed.cellId, + replacementTargetCellId: replacement.cellId, + superseded, + remainingUnregistered: remaining.inProgress + }) +} + +export async function runMultiTargetDeployment(config, overrides = {}) { + const deps = { + commandJson: overrides.commandJson ?? defaultCommandJson, + command: overrides.command ?? defaultCommand, + commandResult: overrides.commandResult ?? defaultCommandResult, + terraform: overrides.terraform, + terraformFenceApply: overrides.terraformFenceApply ?? runTerraformFenceApply, + terraformFenceAdopt: + overrides.terraformFenceAdopt ?? adoptLegacyTerraformFence, + terraformFenceResume: overrides.terraformFenceResume ?? resumeTerraformFence, + terraformFenceRecoverCompleted: + overrides.terraformFenceRecoverCompleted ?? + recoverSupersededCompletedTerraformFence, + terraformFenceAbort: overrides.terraformFenceAbort ?? abortTerraformFenceBeforeApply, + terraformFenceSupersede: + overrides.terraformFenceSupersede ?? abortSupersededTerraformFenceBeforeUpload, + identityToken: overrides.identityToken ?? defaultIdentityToken, + mutationIdentityToken: + overrides.mutationIdentityToken ?? + (() => suppliedFenceMutationIdentityToken()), + fetch: overrides.fetch ?? fetch, + emit: overrides.emit ?? ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)), + now: overrides.now ?? Date.now, + wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + random: overrides.random ?? Math.random, + resolve4: overrides.resolve4 ?? resolve4 + } + const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8')) + const { source, targets } = selectMultiTargetDeployments( + topology, + config.sourceCellId, + config.targetCellIds + ) + const token = deps.identityToken(config.adminAudience) + const mutationToken = + ['fence-source', 'abort-fence-source', 'supersede-target'].includes(config.mode) + ? deps.mutationIdentityToken(config.adminAudience) + : null + if ( + ['fence-source', 'abort-fence-source', 'supersede-target'].includes(config.mode) && + mutationToken === null + ) { + throw new Error('Terraform fence mode requires a broker mutation identity token') + } + const adminPost = createAdminPost( + config, + deps, + (path) => (FENCE_BROKER_MUTATION_ROUTES.has(path) ? mutationToken : token) + ) + if (config.mode === 'abort-fence-source') { + await abortTerraformManagedFence(config, deps, adminPost, source) + return + } + const selectorPost = async (path, body) => + await adminPost(config.directorOrigin, path, body) + const selectorInspection = await inspectAdmissionSelector(selectorPost) + const selectorActive = selectorInspection.selector.generation > 0 + if (config.mode === 'cutover-admission') { + const membership = cutoverMembership(topology, config) + if ( + selectorActive && + JSON.stringify(selectorInspection.selector.membership) !== JSON.stringify(membership) + ) { + throw new Error('selector boundary is already active with different membership') + } + await inspectCutoverCells(topology, config, deps, adminPost, membership) + pruneIncompatibleDirectorRevisions(config, deps) + const director = verifySelectorCompatibleDirector(config, deps) + await inspectCutoverCells(topology, config, deps, adminPost, membership) + const result = await applyExactAdmissionSelector(selectorPost, membership, { + requireBoundary: false, + attemptId: config.selectorAttemptId, + expectedCurrentSelector: selectorInspection.selector + }) + deps.emit({ + event: 'admission_selector_cutover', + generation: result.selector.generation, + membership: result.selector.membership, + ...director + }) + return + } + if (config.mode === 'add-migration-cells') { + if (!selectorActive) throw new Error('admission selector boundary is not active') + pruneIncompatibleDirectorRevisions(config, deps) + const director = verifySelectorCompatibleDirector(config, deps) + if ( + targets.some( + (target) => + target.connectionHardCap === undefined || + target.connectionUnobservedBound === undefined + ) + ) { + throw new Error('migration cells require reviewed connection capacity') + } + const result = await addExactMigrationCells( + selectorPost, + { + attemptId: config.selectorAttemptId, + cells: targets.map((target) => ({ + cellId: target.cellId, + cellUrl: target.origin, + region: target.region, + capacityRequests: target.capacityRequests, + connectionHardCap: target.connectionHardCap, + connectionUnobservedBound: target.connectionUnobservedBound + })) + }, + { expectedCurrentSelector: selectorInspection.selector } + ) + deps.emit({ + event: 'migration_cells_added', + generation: result.selector.generation, + membership: result.selector.membership, + cellIds: targets.map(({ cellId }) => cellId), + ...director + }) + return + } + if (config.mode === 'promote-general-cell') { + if (!selectorActive) throw new Error('admission selector boundary is not active') + const [promoted] = targets + if ( + !promoted || + selectorCellState(selectorInspection.selector, promoted.cellId) !== 'migration-only' + ) { + throw new Error('promoted cell admission must be migration-only') + } + const director = verifyActiveSelectorDirector(config, deps, promoted.cellId) + await inspectGeneralPromotionTarget(config, deps, adminPost, promoted) + const result = await applyExactAdmissionSelector( + selectorPost, + membershipWithStates(selectorInspection.selector, { + [promoted.cellId]: 'general' + }), + { + attemptId: config.selectorAttemptId, + expectedCurrentSelector: selectorInspection.selector + } + ) + deps.emit({ + event: 'migration_cell_promoted_general', + generation: result.selector.generation, + membership: result.selector.membership, + cellId: promoted.cellId, + ...director + }) + return + } + if (config.mode === 'retire-migration-cell') { + if (!selectorActive) throw new Error('admission selector boundary is not active') + const [retiring] = targets + if ( + !retiring || + selectorCellState(selectorInspection.selector, retiring.cellId) !== 'migration-only' + ) { + throw new Error('retired cell admission must be migration-only') + } + pruneIncompatibleDirectorRevisions(config, deps) + const director = verifySelectorCompatibleDirector(config, deps) + const result = await applyExactAdmissionSelector( + selectorPost, + membershipWithStates(selectorInspection.selector, { + [retiring.cellId]: 'existing-only' + }), + { + attemptId: config.selectorAttemptId, + expectedCurrentSelector: selectorInspection.selector + } + ) + deps.emit({ + event: 'migration_cell_retired', + generation: result.selector.generation, + membership: result.selector.membership, + cellId: retiring.cellId, + ...director + }) + return + } + if (config.mode === 'supersede-target') { + await runTargetSupersession( + config, + deps, + adminPost, + source, + targets, + selectorActive ? selectorInspection.selector : null + ) + return + } + const sourceFence = config.mode === 'fence-source' ? sourceMig(config, deps, source) : null + if ( + sourceFence && + ![0, 1].includes(Number(sourceFence.mig.targetSize)) + ) { + throw new Error('source MIG must be fixed-one or already fenced') + } + const fencedResume = sourceFence && Number(sourceFence.mig.targetSize) === 0 ? sourceFence : null + const { sourceProcess, sourceStatus, plannedTargets, sourceAlreadyFenced } = await preflight( + config, + deps, + adminPost, + source, + targets, + fencedResume, + selectorActive ? selectorInspection.selector : null + ) + if (config.mode === 'audit' || config.mode === 'preflight') return + if (config.mode === 'fence-source') { + await fenceSource( + config, + deps, + adminPost, + source, + targets, + sourceProcess, + sourceStatus, + sourceAlreadyFenced + ) + return + } + if (config.mode === 'recover-forward') { + const invalidSelectorAdmission = + selectorActive && + (selectorCellState(selectorInspection.selector, source.cellId) !== 'existing-only' || + plannedTargets.some( + (target) => + selectorCellState(selectorInspection.selector, target.cellId) !== + 'migration-only' + )) + if ( + invalidSelectorAdmission || + (!selectorActive && + (sourceStatus.enabled || plannedTargets.some((target) => !target.status.enabled))) + ) { + throw new Error( + selectorActive + ? 'forward recovery requires existing-only source and migration-only targets' + : 'forward recovery requires disabled source and enabled targets' + ) + } + let coveredSourceConnections = runtimeConnections(sourceProcess, source.cellId) + let recoveryTargets = plannedTargets + for (let pass = 1; pass <= RECOVERY_CATCH_UP_PASSES; pass++) { + await publishMigrations(config, deps, adminPost, recoveryTargets) + if ( + await assertRecoveryPreDrain( + config, + deps, + adminPost, + source, + targets, + selectorPost, + selectorActive ? selectorInspection.selector : null + ) + ) { + break + } + if (pass === RECOVERY_CATCH_UP_PASSES) { + throw new Error('source assignments did not quiesce within bounded recovery catch-up') + } + const catchUp = await preflight( + config, + deps, + adminPost, + source, + targets, + null, + selectorActive ? selectorInspection.selector : null, + coveredSourceConnections + ) + coveredSourceConnections = Math.max( + coveredSourceConnections, + runtimeConnections(catchUp.sourceProcess, source.cellId) + ) + recoveryTargets = catchUp.plannedTargets + } + if (!sourceStatus.draining) { + const activeSourceTransports = + sourceProcess.controls + sourceProcess.splices + sourceProcess.pendingSplices + if (activeSourceTransports > 0) { + const recovery = await adminPost( + config.directorOrigin, + '/v1/admin/drain-attempt-recover-forward', + { + v: 1, + cellId: source.cellId, + cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId), + confirmation: 'RECOVER_LEGACY_DRAIN' + } + ) + await waitForRecoveryTargetOwnership(config, deps, adminPost, targets) + if (recovery.preparedAttempt) { + const attempt = recovery.preparedAttempt + if ( + attempt.state !== 'prepared' || + attempt.cellId !== source.cellId || + attempt.cellIncarnation !== runtimeIncarnation(sourceStatus, source.cellId) || + attempt.plannedGraceMs !== 120_000 || + typeof attempt.attemptId !== 'string' || + typeof attempt.traceValue !== 'string' + ) { + throw new Error('prepared drain recovery state is invalid') + } + const sending = await adminPost( + config.directorOrigin, + '/v1/admin/drain-attempt-send', + { + v: 1, + attemptId: attempt.attemptId, + cellId: source.cellId, + cellIncarnation: attempt.cellIncarnation + } + ) + if ( + sending.attempt?.state !== 'send-may-have-started' || + sending.attempt.shouldSend !== true || + !Number.isSafeInteger(sending.attempt.sendPermitExpiresAt) || + deps.now() >= sending.attempt.sendPermitExpiresAt + ) { + throw new Error('prepared drain recovery send permit unavailable') + } + const receipt = await drainSource( + config, + deps, + token, + source, + attempt.plannedGraceMs, + attempt.traceValue + ) + await adminPost(config.directorOrigin, '/v1/admin/drain-attempt-receipt', { + v: 1, + attemptId: attempt.attemptId, + cellId: source.cellId, + cellIncarnation: attempt.cellIncarnation, + traceValue: attempt.traceValue, + ...receipt + }) + deps.emit({ + event: 'source_prepared_drain_recovered', + sourceCellId: source.cellId + }) + } else if (recovery.shouldSend === true) { + await drainSource(config, deps, token, source, 0) + deps.emit({ event: 'source_recovery_drain_accepted', sourceCellId: source.cellId }) + } else { + const latestSource = await inspectCell(config, deps, adminPost, source) + const expectedIncarnation = runtimeIncarnation(sourceStatus, source.cellId) + if (runtimeIncarnation(latestSource, source.cellId) !== expectedIncarnation) { + throw new Error('source incarnation changed before recovery drain reissue') + } + if (latestSource.draining) { + deps.emit({ + event: 'source_recovery_drain_already_applied', + sourceCellId: source.cellId + }) + } else { + const latestStatuses = await allPairStatuses(config, adminPost, false) + const latestTotals = statusTotals(latestStatuses) + assertLeaseGate(latestStatuses, config.minimumLeaseRemainingMs) + if ( + !hasDurableTargetOwnership(latestTotals) && + !boundedUnregisteredMigrations( + latestTotals, + config.unobservedConnectionBound + ) + ) { + throw new Error('recovery target ownership changed before drain reissue') + } + await drainSource(config, deps, token, source, 0) + deps.emit({ + event: 'source_recovery_drain_reissued_after_non_delivery', + sourceCellId: source.cellId, + ...latestTotals + }) + } + } + } else { + deps.emit({ + event: 'source_recovery_drain_not_needed', + sourceCellId: source.cellId + }) + } + } + await waitForMultiStatus(config, deps, adminPost, targets, false, true) + await waitForRecoveredSourceZero( + config, + deps, + adminPost, + source, + runtimeIncarnation(sourceStatus, source.cellId) + ) + await waitForMultiStatus(config, deps, adminPost, targets, true, true) + deps.emit({ event: 'multi_target_complete', sourceCellId: source.cellId }) + return + } + if ( + selectorActive && + targets.some( + (target) => + selectorCellState(selectorInspection.selector, target.cellId) !== 'migration-only' + ) + ) { + throw new Error('evacuation targets must be migration-only') + } + await runEvacuation( + config, + deps, + adminPost, + token, + source, + targets, + plannedTargets, + sourceStatus, + selectorActive, + selectorPost, + selectorInspection.selector + ) +} + +export async function main(argv = process.argv.slice(2)) { + await runMultiTargetDeployment(parseMultiTargetArguments(argv)) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs b/cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs new file mode 100644 index 00000000000..1ad4e240d56 --- /dev/null +++ b/cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs @@ -0,0 +1,3320 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { + allocateTargetQuotas, + assertCutoverCellReady, + cutoverMembership, + parseMultiTargetArguments, + pruneIncompatibleDirectorRevisions, + runMultiTargetDeployment, + sameBackendServiceResource, + selectMultiTargetDeployments, + verifySelectorCompatibleDirector +} from './deploy-relay-gce-multi-target.mjs' + +const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com' +const digest = (value) => `sha256:${value.repeat(64)}` + +test('matches only exact canonical backend-service resource forms', () => { + const resource = 'projects/onorca-cloud/global/backendServices/orca-cloud-relay-gce-c11' + assert.equal( + sameBackendServiceResource( + `https://www.googleapis.com/compute/v1/${resource}`, + resource + ), + true + ) + assert.equal( + sameBackendServiceResource( + `https://www.googleapis.com/compute/v1/${resource}`, + resource.replace('c11', 'c12') + ), + false + ) + assert.equal( + sameBackendServiceResource( + `https://compute.example/compute/v1/${resource}`, + resource + ), + false + ) +}) + +test('requires only compatible active and rollback director revisions', () => { + let rollbackInventory = ['c1'] + const revision = (minimum = 1, inventory = ['c1']) => ({ + metadata: { + annotations: { 'autoscaling.knative.dev/minScale': String(minimum) } + }, + spec: { + containers: [ + { + image: 'registry.example/relay@sha256:abc', + env: [ + { name: 'ORCA_RELAY_ROLE', value: 'director' }, + { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, + { + name: 'ORCA_RELAY_CELLS_JSON', + value: JSON.stringify(inventory.map((id) => ({ id }))) + } + ] + } + ] + } + }) + let names = ['active', 'rollback'] + const deps = { + command: (args) => { + const revision = args[3] + names = names.filter((name) => name !== revision) + }, + commandJson: (args) => { + if (args.includes('services')) { + return { + status: { + traffic: [ + { percent: 100, revisionName: 'active' }, + { tag: 'selector-rollback', revisionName: 'rollback' } + ] + } + } + } + if (args.includes('list')) { + return names.map((name) => ({ metadata: { name } })) + } + return args.includes('rollback') + ? revision(0, rollbackInventory) + : revision(1) + } + } + const config = { + project: 'project', + directorRegion: 'region', + directorService: 'service', + directorMinimumInstances: 1 + } + assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { + activeRevision: 'active', + rollbackRevision: 'rollback' + }) + rollbackInventory = ['c1', 'c2'] + assert.throws( + () => verifySelectorCompatibleDirector(config, deps), + /director inventories do not match/ + ) + assert.throws( + () => pruneIncompatibleDirectorRevisions(config, deps), + /director compatibility pair failed/ + ) + rollbackInventory = ['c1'] + names = ['active', 'rollback', 'legacy'] + assert.throws( + () => verifySelectorCompatibleDirector(config, deps), + /old or pre-selector director revisions/ + ) + pruneIncompatibleDirectorRevisions(config, deps) + assert.deepEqual(names, ['active', 'rollback']) + assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { + activeRevision: 'active', + rollbackRevision: 'rollback' + }) + names = ['active', 'rollback', null] + assert.throws( + () => verifySelectorCompatibleDirector(config, deps), + /old or pre-selector director revisions/ + ) + assert.throws( + () => pruneIncompatibleDirectorRevisions(config, deps), + /unnamed revision/ + ) +}) + +test('requires the active director to meet the configured floor', () => { + let activeMinimum = 5 + let rollbackMinimum = 0 + const revision = (minimum) => ({ + metadata: { + annotations: { 'autoscaling.knative.dev/minScale': String(minimum) } + }, + spec: { + containers: [ + { + image: 'registry.example/relay@sha256:abc', + env: [ + { name: 'ORCA_RELAY_ROLE', value: 'director' }, + { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, + { + name: 'ORCA_RELAY_CELLS_JSON', + value: JSON.stringify([{ id: 'c1' }]) + } + ] + } + ] + } + }) + const deps = { + command: () => {}, + commandJson: (args) => { + if (args.includes('services')) { + return { + status: { + traffic: [ + { percent: 100, revisionName: 'active' }, + { tag: 'selector-rollback', revisionName: 'rollback' } + ] + } + } + } + if (args.includes('list')) { + return ['active', 'rollback'].map((name) => ({ metadata: { name } })) + } + return args.includes('rollback') ? revision(rollbackMinimum) : revision(activeMinimum) + } + } + const config = { + project: 'project', + directorRegion: 'region', + directorService: 'service', + directorMinimumInstances: 5 + } + + assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { + activeRevision: 'active', + rollbackRevision: 'rollback' + }) + pruneIncompatibleDirectorRevisions(config, deps) + + activeMinimum = 4 + assert.throws( + () => verifySelectorCompatibleDirector(config, deps), + /active selector revision is below the required floor/ + ) + assert.throws( + () => pruneIncompatibleDirectorRevisions(config, deps), + /director compatibility pair failed/ + ) + + // Every comparison against NaN is false, so negating the minimum check rejects it. + activeMinimum = 'warm' + assert.throws( + () => verifySelectorCompatibleDirector(config, deps), + /active selector revision is below the required floor/ + ) + assert.throws( + () => pruneIncompatibleDirectorRevisions(config, deps), + /director compatibility pair failed/ + ) + + activeMinimum = 5 + rollbackMinimum = 1 + assert.throws( + () => verifySelectorCompatibleDirector(config, deps), + /selector rollback revision is not scale-to-zero/ + ) + assert.throws( + () => pruneIncompatibleDirectorRevisions(config, deps), + /director compatibility pair failed/ + ) +}) + +function topology() { + const cell = (id, hostname, initiallyEnabled) => ({ + origin: `https://${hostname}.relay.example.com`, + zone: `us-central1-${hostname}`, + mig_name: `relay-${hostname}`, + instance_group: `https://compute.example/instanceGroups/relay-${hostname}`, + backend_name: `relay-${hostname}`, + backend_id: `https://compute.example/backendServices/relay-${hostname}`, + url_map_name: 'orca-relay', + generation_identity: `https://compute.example/instanceTemplates/relay-${hostname}-abc`, + image: `us-central1-docker.pkg.dev/project/repo/relay@${digest(id)}`, + capacity_requests: 4_000, + connection_hard_cap: 600, + connection_unobserved_bound: 40, + initially_enabled: initiallyEnabled, + fenced: false, + desired_target_size: 1, + target_size: 1 + }) + return { + source: cell('a', 'a', true), + target1: cell('b', 'b', false), + target2: cell('c', 'c', false), + general: cell('d', 'd', true) + } +} + +function withTopology(operation, value = topology()) { + const directory = mkdtempSync(join(tmpdir(), 'relay-gce-multi-')) + const file = join(directory, 'topology.json') + writeFileSync(file, JSON.stringify(value)) + return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true })) +} + +function config(topologyFile, mode = 'preflight') { + const selectorMutation = [ + 'cutover-admission', + 'add-migration-cells', + 'promote-general-cell', + 'retire-migration-cell' + ].includes(mode) + return { + project: 'test-project', + directorOrigin: 'https://relay.example.com', + adminAudience: 'https://relay.example.com/v1/admin/drain', + topologyFile, + sourceCellId: 'source', + targetCellIds: ['promote-general-cell', 'retire-migration-cell'].includes(mode) + ? ['target1'] + : ['target1', 'target2'], + generalCellIds: mode === 'cutover-admission' ? ['general'] : [], + directorRegion: selectorMutation ? 'us-central1' : undefined, + directorService: selectorMutation ? 'relay-director' : undefined, + directorMinimumInstances: selectorMutation ? 1 : undefined, + selectorAttemptId: + mode === 'add-migration-cells' + ? 'add_cells_test' + : mode === 'promote-general-cell' + ? 'promote_cell_test' + : mode === 'retire-migration-cell' + ? 'retire_cell_test' + : undefined, + unobservedConnectionBound: + selectorMutation || mode === 'fence-source' ? 40 : undefined, + failedTargetCellId: mode === 'supersede-target' ? 'target1' : undefined, + replacementTargetCellId: mode === 'supersede-target' ? 'target2' : undefined, + runtimeServiceAccount, + environment: 'production', + fenceCommit: 'a'.repeat(40), + terraformDir: 'infra/terraform', + terraformVarFile: 'environments/production.tfvars', + mode, + batchSize: 2, + connectionCeiling: mode === 'fence-source' ? 600 : 6, + minimumLeaseRemainingMs: 600_000, + drainGraceMs: 120_000, + pollIntervalMs: 1, + timeoutMs: 1_000 + } +} + +function response(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +function harness({ + leaseRemainingMs = 900_000, + refreshedLeaseRemainingMs = leaseRemainingMs, + failAfterDrain = false, + failEvacuationStatus = false, + fence = false, + allowPreFenceCompletion = false, + loseResizeResponse = false, + resumeNeedsPreApply = false, + loseDrainSendResponse = false, + loseDrainBeforeAccept = false, + preparedDrainAttempt = false, + recoverableDrain = false, + recoveryAlreadyAttempted = false, + preexistingRegisteredMigrations = 0, + supersede = false, + cleanupBeforeSupersession = false, + offlineTargetMigrations = 0, + offlineTargetMigrationsByTarget = {}, + unregisteredTargetMigrations = 0, + registeredSourceActive = 0, + recoveryRegistrationDelayReads = 0, + failedTargetEnabled = false, + replacementHeartbeatFresh = true, + replacementRuntimeCellUrl, + legacySource = false, + legacyMetricAgeMs = 0, + selectorMembership = null, + capacitySourceAssignments = [], + capacityRequiredTargetUnits = [], + headroomFailureTarget = null, + sourceAssignments = fence ? 0 : 4, + sourceRequiredTargetUnits = fence ? 0 : 8, + sourceObservedRequests = null, + sourceOutgoingMigrations = null, + existingFenceAttempt = false, + alreadyFencedSource = false, + supersededFenceAttempt = false, + completedFenceAttempt = false, + activeDirectorMinimum = 1, + misroutedCell = null, + routeOverrideCell = null, + frontendMisbound = false, + templateHardCap = 600, + templateUnobservedBound = 40, + directorHardCap = templateHardCap, + directorUnobservedBound = templateUnobservedBound, + directorCapacityOverrides = {}, + liveMigTemplateOverrides = {}, + topologyValue = topology() +} = {}) { + const directorCapacity = (cellId) => { + const hardCap = directorCapacityOverrides[cellId]?.hardCap ?? directorHardCap + const unobservedBound = + directorCapacityOverrides[cellId]?.unobservedBound ?? directorUnobservedBound + return { + hardCap, + controlRebindReserve: 100, + ordinaryConnectionLimit: hardCap - 100, + unobservedBound, + normalAdmissionPause: hardCap - 100 - unobservedBound + } + } + const cells = { + source: { + enabled: !fence && !supersede, + assignments: 4, + activityLeases: fence ? 0 : 4, + totalConnections: fence ? 1 : 5, + controls: fence ? 0 : 4, + observedRequests: sourceObservedRequests ?? (fence ? 0 : 4), + heartbeatFresh: !alreadyFencedSource, + draining: fence + }, + target1: { + enabled: failedTargetEnabled || (fence && !supersede), + assignments: fence ? 2 : 0, + activityLeases: fence ? 2 : 0, + totalConnections: fence ? 2 : 0, + controls: fence ? 2 : 0, + heartbeatFresh: !supersede, + draining: false + }, + target2: { + enabled: fence && !supersede, + assignments: fence ? 2 : 0, + activityLeases: fence ? 2 : 0, + totalConnections: fence ? 2 : 0, + controls: fence ? 2 : 0, + heartbeatFresh: replacementHeartbeatFresh, + draining: false + }, + general: { + enabled: true, + assignments: 0, + activityLeases: 0, + totalConnections: 0, + controls: 0, + heartbeatFresh: true, + draining: false + } + } + for (const cellId of Object.keys(topologyValue)) { + cells[cellId] ??= { + enabled: fence && !supersede, + assignments: fence ? 2 : 0, + activityLeases: fence ? 2 : 0, + totalConnections: fence ? 2 : 0, + controls: fence ? 2 : 0, + heartbeatFresh: true, + draining: false + } + } + const events = [] + const stateChanges = [] + const batches = [] + const publishedMigrations = new Map() + const runtimeInspections = new Map() + let drained = fence + let remainingSourceAssignments = sourceAssignments + let remainingRequiredTargetUnits = sourceRequiredTargetUnits + const migSizes = Object.fromEntries( + Object.keys(topologyValue).map((cellId) => [ + cellId, + cellId === 'source' && alreadyFencedSource + ? 0 + : cellId === 'target1' && completedFenceAttempt + ? 0 + : 1 + ]) + ) + const instanceCounts = { ...migSizes } + let supersessionRemaining = supersede ? 2 : 0 + let remainingOfflineTargetMigrations = offlineTargetMigrations + const initialOfflineTargetMigrationsByTarget = new Map( + Object.entries(offlineTargetMigrationsByTarget) + ) + const remainingOfflineTargetMigrationsByTarget = new Map( + initialOfflineTargetMigrationsByTarget + ) + const targetMigrationTotal = (cellId) => + initialOfflineTargetMigrationsByTarget.has(cellId) + ? initialOfflineTargetMigrationsByTarget.get(cellId) + : 2 + const remainingOfflineTargetMigrationCount = (cellId) => + remainingOfflineTargetMigrationsByTarget.has(cellId) + ? remainingOfflineTargetMigrationsByTarget.get(cellId) + : remainingOfflineTargetMigrations + const clearOfflineTargetMigrations = (cellId) => { + if (remainingOfflineTargetMigrationsByTarget.has(cellId)) { + remainingOfflineTargetMigrationsByTarget.set(cellId, 0) + return + } + remainingOfflineTargetMigrations = 0 + } + let drainReceiptRecorded = recoverableDrain + let drainSendStarted = false + let recoveryDrainPrepared = recoveryAlreadyAttempted + let recoveryLeasesRefreshed = false + const recoveryRegistrationReads = new Map() + let headroomFailureInjected = false + let currentLeaseRemainingMs = leaseRemainingMs + const drainGraces = [] + const timeline = [] + let fencedCompletions = 0 + let resizeAttempts = 0 + let capacityReads = 0 + let addedCells = [] + let fenceAttested = false + let fenceAttempt = existingFenceAttempt || supersededFenceAttempt || completedFenceAttempt + ? { + attemptId: '44444444-4444-4444-8444-444444444444', + environment: 'production', + cellId: 'source', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + migName: topologyValue.source.mig_name, + instanceGroup: topologyValue.source.instance_group, + generationIdentity: topologyValue.source.generation_identity, + fenceCommit: (supersededFenceAttempt || completedFenceAttempt ? 'b' : 'a').repeat(40), + planSha256: 'd'.repeat(64), + planObjectName: + 'terraform/state/relay-fence-plans/production/44444444-4444-4444-8444-444444444444.tfplan', + ...(supersededFenceAttempt && !completedFenceAttempt + ? {} + : { planObjectGeneration: '123456789' }), + varFileSha256: 'e'.repeat(64), + terraformStateLineage: '55555555-5555-4555-8555-555555555555', + terraformStateSerial: 7, + terraformStateObjectGeneration: '987654321', + terraformStateObjectSha256: 'f'.repeat(64), + requestReason: + 'orca-relay-fence/44444444-4444-4444-8444-444444444444', + createdAt: Date.now(), + expiresAt: Date.now() + 3_600_000, + ...(completedFenceAttempt + ? { + applyStartedAt: 101, + applyInvocations: [ + { + invocationId: '66666666-6666-4666-8666-666666666666', + requestReason: + 'orca-relay-fence/44444444-4444-4444-8444-444444444444/66666666-6666-4666-8666-666666666666', + startedAt: 101 + } + ] + } + : {}) + } + : null + let selector = selectorMembership + ? { generation: 1, attemptId: 'existing-selector', membership: selectorMembership } + : null + const selectorIntents = new Map() + const cellForOrigin = (origin) => { + const entry = Object.entries(topologyValue).find(([, value]) => value.origin === origin) + return entry?.[0] + } + const cellForMig = (name) => { + const entry = Object.entries(topologyValue).find(([, value]) => value.mig_name === name) + return entry?.[0] + } + const commandJson = (args) => { + if (args[0] === 'run') { + if (args.includes('services')) { + return { + status: { + traffic: [ + { percent: 100, revisionName: 'active' }, + { percent: 0, tag: 'selector-rollback', revisionName: 'rollback' } + ] + } + } + } + if (args.includes('list')) { + return ['active', 'rollback'].map((name) => ({ metadata: { name } })) + } + const revisionName = args[3] + return { + metadata: { + annotations: { + 'autoscaling.knative.dev/minScale': + revisionName === 'rollback' ? '0' : String(activeDirectorMinimum) + } + }, + spec: { + containers: [ + { + image: 'registry.example/relay@sha256:abc', + env: [ + { name: 'ORCA_RELAY_ROLE', value: 'director' }, + { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, + { + name: 'ORCA_RELAY_CELLS_JSON', + value: JSON.stringify( + Object.keys(topologyValue).map((id) => ({ id })) + ) + } + ] + } + ] + } + } + } + if (args.includes('instance-templates')) { + const templateName = args[args.indexOf('describe') + 1] + const expected = Object.values(topologyValue).find((cell) => + cell.generation_identity.endsWith(`/instanceTemplates/${templateName}`) + ) + return { + selfLink: expected.generation_identity, + properties: { + metadata: { + items: [ + { + key: 'startup-script', + value: [ + `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${expected.image.split('@')[1]}'`, + ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${templateHardCap}'`, + ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${templateUnobservedBound}'`, + `docker pull '${expected.image}'`, + `docker run '${expected.image}'` + ].join('\n') + } + ] + } + } + } + } + if (args[0] === 'logging') { + return [ + { + timestamp: new Date(Date.now() - legacyMetricAgeMs).toISOString(), + jsonPayload: { + totalConnections: cells.source.totalConnections, + preAuthConnections: 0, + controls: cells.source.controls, + splices: 0, + pendingSplices: 0, + queuedBytes: 0 + } + } + ] + } + const describeIndex = args.indexOf('describe') + const listIndex = args.indexOf('list-instances') + const name = args[describeIndex >= 0 ? describeIndex + 1 : listIndex + 1] + const migCell = cellForMig(name) + if (args.includes('list-instances')) { + return instanceCounts[migCell] === 0 + ? [] + : [ + { + instance: `https://compute.example/instances/${name}-vm`, + instanceStatus: 'RUNNING', + currentAction: 'NONE', + version: { + name: 'primary', + instanceTemplate: topologyValue[migCell].generation_identity + } + } + ] + } + if (args.includes('instance-groups')) { + return { + targetSize: migSizes[migCell], + instanceTemplate: + liveMigTemplateOverrides[migCell] ?? topologyValue[migCell].generation_identity, + updatePolicy: { + replacementMethod: 'RECREATE', + maxSurge: { fixed: 0 }, + maxUnavailable: { fixed: 1 } + } + } + } + if (args.includes('instances')) { + return { + networkInterfaces: [{ networkIP: '10.42.0.2' }], + serviceAccounts: [{ email: runtimeServiceAccount }] + } + } + if (args.includes('target-https-proxies')) { + return { + name: 'orca-relay', + selfLink: + 'https://www.googleapis.com/compute/v1/projects/test-project/global/targetHttpsProxies/orca-relay', + urlMap: frontendMisbound + ? 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/other' + : 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/orca-relay' + } + } + if (args.includes('forwarding-rules')) { + return { + name: 'orca-relay', + target: + 'https://www.googleapis.com/compute/v1/projects/test-project/global/targetHttpsProxies/orca-relay', + IPAddress: '203.0.113.10', + portRange: '443-443', + loadBalancingScheme: 'EXTERNAL_MANAGED' + } + } + if (args.includes('addresses')) { + return { name: 'orca-relay', address: '203.0.113.10' } + } + if (args.includes('url-maps')) { + return { + name: 'orca-relay', + selfLink: + 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/orca-relay', + hostRules: Object.entries(topologyValue).map(([cellId, cell]) => ({ + hosts: [new URL(cell.origin).hostname], + pathMatcher: cellId + })), + pathMatchers: Object.entries(topologyValue).map(([cellId, cell]) => ({ + name: cellId, + defaultService: + cellId === misroutedCell ? topologyValue.general.backend_id : cell.backend_id, + ...(cellId === routeOverrideCell + ? { + pathRules: [ + { paths: ['/v1/*'], service: topologyValue.general.backend_id } + ] + } + : {}) + })) + } + } + const id = Object.entries(topologyValue).find(([, value]) => value.backend_name === name)?.[0] + return { + protocol: 'HTTP', + timeoutSec: 86_400, + backends: [{ group: topologyValue[id].instance_group }] + } + } + const fetch = async (url, options = {}) => { + const parsed = new URL(url) + if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true }) + const body = JSON.parse(options.body ?? '{}') + if (parsed.pathname === '/v1/admin/runtime-status') { + const id = cellForOrigin(parsed.origin) + const cell = cells[id] + const capacity = directorCapacity(id) + runtimeInspections.set(id, (runtimeInspections.get(id) ?? 0) + 1) + return response({ + v: 1, + role: 'cell', + cellId: id, + cellUrl: topologyValue[id].origin, + imageDigest: topologyValue[id].image.split('@')[1], + draining: cell.draining, + connectionCapacity: { + ...capacity + }, + runtime: + legacySource && id === 'source' + ? null + : { + totalConnections: cell.totalConnections, + preAuthConnections: 0, + enforcedConnectionUnits: cell.totalConnections, + controls: cell.controls, + splices: 0, + pendingSplices: 0, + queuedBytes: 0 + } + }) + } + if (parsed.pathname === '/v1/admin/cell-status') { + const cell = cells[body.cellId] + const capacity = directorCapacity(body.cellId) + return response({ + v: 1, + status: { + cellId: body.cellId, + cellUrl: topologyValue[body.cellId].origin, + enabled: cell.enabled, + assignments: cell.assignments, + activityLeases: cell.activityLeases, + activityRequestUnits: cell.activityLeases, + reservedRequests: cell.activityLeases, + connectionCapacity: { + ...capacity, + observedConnections: cell.totalConnections, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: cell.totalConnections, + pendingControlReservations: 0, + heartbeatFresh: cell.heartbeatFresh + }, + outgoingMigrations: + sourceOutgoingMigrations ?? + (fence + ? Object.keys(topologyValue) + .filter((cellId) => cellId !== 'source' && cellId !== 'general') + .reduce((total, cellId) => total + targetMigrationTotal(cellId), 0) + : 0), + incomingMigrations: 0, + runtime: { + cellUrl: + body.cellId === 'target2' && replacementRuntimeCellUrl + ? replacementRuntimeCellUrl + : topologyValue[body.cellId].origin, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + ready: true, + heartbeatFresh: cell.heartbeatFresh, + observedRequests: cell.observedRequests ?? cell.controls + } + } + }) + } + if (parsed.pathname === '/v1/admin/admission-selector/status') { + selector ??= { + generation: 0, + attemptId: null, + membership: { + existingOnly: Object.keys(cells).filter((cellId) => !cells[cellId].enabled), + migrationOnly: [], + general: Object.keys(cells).filter((cellId) => cells[cellId].enabled) + } + } + return response({ + v: 1, + selector, + intent: body.attemptId ? selectorIntents.get(body.attemptId) ?? null : null + }) + } + if (parsed.pathname === '/v1/admin/admission-selector/apply') { + selector = { + generation: body.expectedGeneration + 1, + attemptId: body.attemptId, + membership: body.membership + } + selectorIntents.set(body.attemptId, { + attemptId: body.attemptId, + expectedGeneration: body.expectedGeneration, + intendedGeneration: selector.generation, + membership: body.membership, + state: 'committed' + }) + return response({ v: 1, changed: true, selector }) + } + if (parsed.pathname === '/v1/admin/admission-selector/add-migration-cells') { + addedCells = body.cells + selector = { + generation: body.expectedGeneration + 1, + attemptId: body.attemptId, + membership: { + ...selector.membership, + migrationOnly: [ + ...selector.membership.migrationOnly, + ...body.cells.map(({ cellId }) => cellId) + ].sort() + } + } + selectorIntents.set(body.attemptId, { + attemptId: body.attemptId, + expectedGeneration: body.expectedGeneration, + intendedGeneration: selector.generation, + membership: selector.membership, + state: 'committed' + }) + return response({ v: 1, changed: true, selector }) + } + if (parsed.pathname === '/v1/admin/evacuation-capacity') { + const read = capacityReads++ + return response({ + v: 1, + sourceAssignments: capacitySourceAssignments[read] ?? remainingSourceAssignments, + requiredTargetUnits: + capacityRequiredTargetUnits[read] ?? remainingRequiredTargetUnits, + availableTargetUnits: 4_000 + }) + } + if (parsed.pathname === '/v1/admin/cell-state') { + cells[body.cellId].enabled = body.enabled + stateChanges.push([body.cellId, body.enabled]) + return response({ ok: true }) + } + if (parsed.pathname === '/v1/admin/drain-attempt-prepare') { + return response({ v: 1, state: 'prepared' }) + } + if (parsed.pathname === '/v1/admin/drain-attempt-send') { + const shouldSend = !drainSendStarted + drainSendStarted = true + if (loseDrainSendResponse) throw new Error('injected_send_transition_response_loss') + return response({ + v: 1, + attempt: { + state: 'send-may-have-started', + shouldSend, + sendPermitExpiresAt: Date.now() + 30_000 + } + }) + } + if (parsed.pathname === '/v1/admin/drain-attempt-receipt') { + drainReceiptRecorded = true + return response({ v: 1, attempt: { state: 'application-receipt' } }) + } + if (parsed.pathname === '/v1/admin/drain-attempt-recover-forward') { + recoveryLeasesRefreshed = true + currentLeaseRemainingMs = refreshedLeaseRemainingMs + if (!drainReceiptRecorded) { + if (preparedDrainAttempt && !drainSendStarted) { + return response({ + v: 1, + shouldSend: false, + retryAfter: Date.now(), + preparedAttempt: { + attemptId: '33333333-3333-4333-8333-333333333333', + cellId: 'source', + cellIncarnation: '11111111-1111-4111-8111-111111111111', + traceValue: '44444444-4444-4444-8444-444444444444', + plannedGraceMs: 120_000, + state: 'prepared' + } + }) + } + return response({ error: 'drain_application_receipt_missing' }, 409) + } + const shouldSend = !recoveryDrainPrepared + recoveryDrainPrepared = true + return response({ v: 1, shouldSend, retryAfter: Date.now() }) + } + if (parsed.pathname === '/v1/admin/evacuate-cell') { + if (body.targetCellId === headroomFailureTarget && !headroomFailureInjected) { + headroomFailureInjected = true + const partiallyStarted = Math.min(1, remainingSourceAssignments) + publishedMigrations.set( + body.targetCellId, + (publishedMigrations.get(body.targetCellId) ?? 0) + partiallyStarted + ) + remainingSourceAssignments -= partiallyStarted + return response({ error: 'relay_connection_headroom_exhausted' }, 409) + } + const started = Math.min(body.limit, remainingSourceAssignments) + batches.push([body.targetCellId, started]) + publishedMigrations.set( + body.targetCellId, + (publishedMigrations.get(body.targetCellId) ?? 0) + started + ) + remainingSourceAssignments -= started + if (remainingSourceAssignments === 0) remainingRequiredTargetUnits = 0 + return response({ v: 1, started }) + } + if (parsed.pathname === '/v1/admin/cell-fence-attest') { + fenceAttested = true + return response({ v: 1, cellId: body.cellId, expiresAt: Date.now() + 300_000 }) + } + if (parsed.pathname === '/v1/admin/cell-fence-adopt-legacy') { + fenceAttested = true + return response({ v: 1, cellId: body.cellId, expiresAt: Date.now() + 300_000 }) + } + if (parsed.pathname === '/v1/admin/cell-fence-commit-legacy-adoption') { + return response({ v: 1, cellId: body.cellId, committed: true }) + } + if (parsed.pathname === '/v1/admin/cell-fence-attempt-prepare') { + fenceAttempt = body + return response({ v: 1, attempt: body }) + } + if (parsed.pathname === '/v1/admin/cell-fence-attempt-start') { + fenceAttempt = { ...body, applyStartedAt: Date.now() } + return response({ + v: 1, + attempt: fenceAttempt, + invocation: { + invocationId: body.invocationId, + requestReason: body.invocationRequestReason, + startedAt: fenceAttempt.applyStartedAt + } + }) + } + if (parsed.pathname === '/v1/admin/cell-fence-attempt-operation') { + fenceAttempt = body + return response({ + v: 1, + attempt: body, + invocation: { + invocationId: body.invocationId, + requestReason: body.invocationRequestReason, + startedAt: Date.now(), + gceOperation: body.gceOperation + } + }) + } + if (parsed.pathname === '/v1/admin/cell-fence-attempt-status') { + return response({ v: 1, attempt: fenceAttempt }) + } + if (parsed.pathname === '/v1/admin/cell-fence-attempt-abort') { + fenceAttempt = { ...body, abortedAt: Date.now() } + return response({ v: 1, attempt: fenceAttempt }) + } + if (parsed.pathname === '/v1/admin/evacuation-status') { + if (failEvacuationStatus) { + return response({ error: 'injected_status_failure' }, 500) + } + const completed = + body.completeReady && (!fence || fenceAttested || allowPreFenceCompletion) + if (completed && fenceAttested) clearOfflineTargetMigrations(body.targetCellId) + const migrationTotal = targetMigrationTotal(body.targetCellId) + const remainingOfflineTargetMigrationsForCell = + remainingOfflineTargetMigrationCount(body.targetCellId) + const supersededTarget = supersede && body.targetCellId === 'target1' + const recoveryRegistrationRead = + recoveryRegistrationReads.get(body.targetCellId) ?? 0 + if (recoveryLeasesRefreshed) { + recoveryRegistrationReads.set(body.targetCellId, recoveryRegistrationRead + 1) + } + const recoveryRegistrationSettled = + recoveryRegistrationRead >= recoveryRegistrationDelayReads + const remainingMigrations = supersededTarget + ? supersessionRemaining + : completed + ? remainingOfflineTargetMigrationsForCell + unregisteredTargetMigrations + : migrationTotal + const registeredMigrations = supersededTarget + ? supersessionRemaining + : drained + ? remainingMigrations - unregisteredTargetMigrations + : Math.min( + remainingMigrations, + preexistingRegisteredMigrations + + (recoveryLeasesRefreshed && recoveryRegistrationSettled + ? publishedMigrations.get(body.targetCellId) ?? 0 + : 0) + ) + const completableMigrations = + drained && !completed + ? migrationTotal - + remainingOfflineTargetMigrationsForCell - + unregisteredTargetMigrations - + registeredSourceActive + : recoveryLeasesRefreshed + ? Math.max( + 0, + registeredMigrations - + remainingOfflineTargetMigrationsForCell - + registeredSourceActive + ) + : 0 + if (completed) { + if (failAfterDrain) return response({ error: 'injected_completion_failure' }, 500) + if (fenceAttested) fencedCompletions++ + cells.source.activityLeases = 0 + for (const targetCellId of Object.keys(cells).filter((id) => id !== 'source')) { + cells[targetCellId].activityLeases = 2 + } + } + timeline.push({ + kind: 'evacuation_status', + targetCellId: body.targetCellId, + completeReady: body.completeReady, + inProgress: remainingMigrations, + registeredTargetInactive: remainingOfflineTargetMigrationsForCell + }) + return response({ + v: 1, + inProgress: remainingMigrations, + oldestExpiresAt: + remainingMigrations === 0 ? null : Date.now() + currentLeaseRemainingMs, + oldestRemainingMs: + remainingMigrations === 0 ? null : currentLeaseRemainingMs, + targetRegistered: registeredMigrations, + registeredSourceActive, + registeredCompletable: completableMigrations, + registeredTargetInactive: remainingOfflineTargetMigrationsForCell, + completed: + completed + ? migrationTotal - + remainingOfflineTargetMigrationsForCell - + unregisteredTargetMigrations + : 0, + blocked: completed ? remainingOfflineTargetMigrationsForCell : 0, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 + }) + } + if (parsed.pathname === '/v1/admin/migration-supersede-cell') { + const superseded = cleanupBeforeSupersession ? 0 : supersessionRemaining + supersessionRemaining = 0 + return response({ v: 1, superseded }) + } + if (parsed.pathname === '/v1/admin/drain') { + drainGraces.push(body.graceMs) + if (loseDrainBeforeAccept && body.graceMs === 120_000) { + throw new Error('injected_drain_response_loss') + } + drained = true + cells.source.draining = true + cells.source.activityLeases = 0 + cells.source.controls = 0 + for (const targetCellId of Object.keys(cells).filter((id) => id !== 'source')) { + cells[targetCellId].totalConnections = 2 + } + return response({ ok: true }) + } + return response({ error: 'unexpected_request' }, 500) + } + return { + overrides: { + commandJson, + terraform: (args) => { + if (args.includes('console')) return 'true\n' + if (args.includes('plan')) return '' + if (args.includes('show') && args.length > 3) { + return JSON.stringify({ resource_changes: [] }) + } + if (args.includes('show')) { + return JSON.stringify({ + values: { + root_module: { + resources: [ + { + address: + 'google_compute_instance_group_manager.relay_gce_cell["source"]', + values: { + name: topologyValue.source.mig_name, + zone: topologyValue.source.zone, + instance_group: topologyValue.source.instance_group, + target_size: migSizes.source, + version: [ + { instance_template: topologyValue.source.generation_identity } + ] + } + } + ] + } + } + }) + } + throw new Error(`unexpected terraform command: ${args.join(' ')}`) + }, + command: (args) => { + throw new Error(`unexpected direct gcloud mutation: ${args.join(' ')}`) + }, + terraformFenceApply: async (fenceConfig, callbacks) => { + const attempt = { + attemptId: '44444444-4444-4444-8444-444444444444', + environment: fenceConfig.environment, + cellId: fenceConfig.cell.cellId, + cellIncarnation: fenceConfig.cellIncarnation, + migName: fenceConfig.cell.migName, + instanceGroup: fenceConfig.cell.instanceGroup, + generationIdentity: fenceConfig.cell.generationIdentity, + fenceCommit: fenceConfig.fenceCommit, + planSha256: 'd'.repeat(64), + planObjectName: + `terraform/state/relay-fence-plans/${fenceConfig.environment}/44444444-4444-4444-8444-444444444444.tfplan`, + planObjectGeneration: '123456789', + varFileSha256: 'e'.repeat(64), + terraformStateLineage: '55555555-5555-4555-8555-555555555555', + terraformStateSerial: 7, + terraformStateObjectGeneration: '987654321', + terraformStateObjectSha256: 'f'.repeat(64), + requestReason: + 'orca-relay-fence/44444444-4444-4444-8444-444444444444' + } + await callbacks.prepareAttempt(attempt) + await callbacks.preApplyGuard() + const invocation = { + invocationId: '66666666-6666-4666-8666-666666666666', + requestReason: + `${attempt.requestReason}/66666666-6666-4666-8666-666666666666` + } + await callbacks.markApplyStarted(attempt, invocation) + const cellId = fenceConfig.cell.cellId + migSizes[cellId] = 0 + instanceCounts[cellId] = 0 + cells[cellId].heartbeatFresh = false + resizeAttempts++ + if (loseResizeResponse && resizeAttempts === 1) { + throw new Error('injected_resize_response_loss') + } + const completed = { ...attempt, gceOperation: 'operation-1' } + await callbacks.markOperation(completed, invocation) + await callbacks.postApplyGuard(attempt.cellIncarnation) + await callbacks.attest(completed) + }, + terraformFenceAdopt: async (fenceConfig, callbacks) => { + assert.equal(await callbacks.loadAttempt(), null) + await callbacks.assertCommittedFenceSet() + await callbacks.preApplyGuard() + await callbacks.assertStateFenced() + await callbacks.postApplyGuard(fenceConfig.cellIncarnation) + assert.equal(await callbacks.loadAttempt(), null) + await callbacks.attest(fenceConfig.cellIncarnation) + await callbacks.postApplyGuard(fenceConfig.cellIncarnation) + await callbacks.commitAdoption(fenceConfig.cellIncarnation) + callbacks.emit({ + event: 'terraform_cell_fence_legacy_adopted', + cellId: fenceConfig.cell.cellId + }) + }, + terraformFenceResume: async (fenceConfig, callbacks) => { + const attempt = await callbacks.loadAttempt() + if (resumeNeedsPreApply) await callbacks.preApplyGuard() + const completed = { + ...attempt, + cellIncarnation: fenceConfig.cellIncarnation, + gceOperation: 'operation-1' + } + await callbacks.postApplyGuard(completed.cellIncarnation) + await callbacks.attest(completed) + }, + terraformFenceRecoverCompleted: async ( + fenceConfig, + callbacks, + recovery + ) => { + const attempt = await callbacks.loadAttempt() + callbacks.emit({ + event: 'terraform_cell_fence_completed_attempt_recovered', + cellId: fenceConfig.cell.cellId, + attemptId: attempt.attemptId, + gceOperation: recovery.gceOperation + }) + }, + terraformFenceAbort: async (fenceConfig, callbacks) => { + await callbacks.abortAttempt() + callbacks.emit({ + event: 'terraform_fence_aborted_before_apply', + cellId: fenceConfig.cell.cellId + }) + }, + terraformFenceSupersede: async (fenceConfig, callbacks) => { + const attempt = await callbacks.loadAttempt() + await callbacks.abortAttempt(attempt) + callbacks.emit({ + event: 'terraform_fence_superseded_before_upload', + cellId: fenceConfig.cell.cellId, + previousFenceCommit: attempt.fenceCommit, + fenceCommit: fenceConfig.fenceCommit + }) + }, + identityToken: () => 'aaa.bbb.ccc', + mutationIdentityToken: () => 'ddd.eee.fff', + resolve4: async () => ['203.0.113.10'], + fetch, + emit: (event) => { + events.push(event) + timeline.push({ kind: 'event', event }) + }, + wait: async () => undefined, + random: () => 0 + }, + batches, + events, + stateChanges, + drainGraces, + timeline, + fencedCompletions: () => fencedCompletions, + capacityReads: () => capacityReads, + selector: () => selector, + addedCells: () => addedCells, + runtimeInspections + } +} + +test('parses deterministic target sets with single-cell selector exceptions', () => { + const common = [ + '--project', + 'project', + '--director-origin', + 'https://relay.example.com', + '--admin-audience', + 'https://relay.example.com/v1/admin/drain', + '--topology-file', + 'topology.json', + '--source-cell-id', + 'source', + '--runtime-service-account', + runtimeServiceAccount, + '--mode', + 'preflight' + ] + assert.deepEqual( + parseMultiTargetArguments([...common, '--target-cell-ids', 'target2,target1']).targetCellIds, + ['target1', 'target2'] + ) + assert.throws( + () => parseMultiTargetArguments([...common, '--target-cell-ids', 'target1']), + /at least 2/ + ) + const cutover = [ + ...common.slice(0, -2), + '--mode', + 'cutover-admission', + '--target-cell-ids', + 'target1,target2', + '--general-cell-ids', + 'general', + '--director-region', + 'us-central1', + '--director-service', + 'relay-director', + '--director-min-instances', + '5' + ] + assert.throws( + () => parseMultiTargetArguments(cutover), + /unobserved-connection-bound/ + ) + assert.equal( + parseMultiTargetArguments([ + ...cutover, + '--unobserved-connection-bound', + '40' + ]).unobservedConnectionBound, + 40 + ) + const recovery = [ + ...common.slice(0, -2), + '--mode', + 'recover-forward', + '--target-cell-ids', + 'target1,target2' + ] + assert.throws( + () => parseMultiTargetArguments(recovery), + /unobserved-connection-bound/ + ) + assert.equal( + parseMultiTargetArguments([ + ...recovery, + '--unobserved-connection-bound', + '60' + ]).unobservedConnectionBound, + 60 + ) + const fenceSource = [ + ...common.slice(0, -2), + '--mode', + 'fence-source', + '--target-cell-ids', + 'target1,target2', + '--fence-commit', + 'a'.repeat(40) + ] + assert.throws( + () => parseMultiTargetArguments(fenceSource), + /unobserved-connection-bound/ + ) + assert.equal( + parseMultiTargetArguments([ + ...fenceSource, + '--unobserved-connection-bound', + '60' + ]).unobservedConnectionBound, + 60 + ) + const addCells = [ + ...common.slice(0, -2), + '--mode', + 'add-migration-cells', + '--target-cell-ids', + 'target1,target2', + '--director-region', + 'us-central1', + '--director-service', + 'relay-director', + '--director-min-instances', + '5', + '--unobserved-connection-bound', + '40' + ] + assert.throws( + () => parseMultiTargetArguments(addCells), + /selector-attempt-id/ + ) + assert.equal( + parseMultiTargetArguments([ + ...addCells, + '--target-cell-ids', + 'target1', + '--selector-attempt-id', + 'add_cells_parse' + ]).targetCellIds.length, + 1 + ) + const retireCell = [ + ...common.slice(0, -2), + '--mode', + 'retire-migration-cell', + '--target-cell-ids', + 'target1', + '--director-region', + 'us-central1', + '--director-service', + 'relay-director', + '--director-min-instances', + '5', + '--selector-attempt-id', + 'retire_cell_parse' + ] + assert.equal( + parseMultiTargetArguments(retireCell).mode, + 'retire-migration-cell' + ) + assert.throws( + () => + parseMultiTargetArguments([ + ...retireCell, + '--target-cell-ids', + 'target1,target2' + ]), + /exactly one/ + ) + const promoteCell = [ + ...common.slice(0, -2), + '--mode', + 'promote-general-cell', + '--target-cell-ids', + 'target1', + '--director-region', + 'us-central1', + '--director-service', + 'relay-director', + '--director-min-instances', + '5', + '--selector-attempt-id', + 'promote_cell_parse' + ] + assert.equal( + parseMultiTargetArguments(promoteCell).mode, + 'promote-general-cell' + ) + assert.throws( + () => + parseMultiTargetArguments([ + ...promoteCell, + '--target-cell-ids', + 'target1,target2' + ]), + /exactly one/ + ) +}) + +test('requires a complete exact completed-fence recovery pin set', () => { + const args = [ + '--project', + 'project', + '--director-origin', + 'https://relay.example.com', + '--admin-audience', + 'https://relay.example.com/v1/admin/drain', + '--topology-file', + 'topology.json', + '--source-cell-id', + 'source', + '--target-cell-ids', + 'target1,target2', + '--runtime-service-account', + runtimeServiceAccount, + '--mode', + 'supersede-target', + '--failed-target-cell-id', + 'target1', + '--replacement-target-cell-id', + 'target2', + '--fence-commit', + 'a'.repeat(40), + '--completed-fence-attempt-id', + '44444444-4444-4444-8444-444444444444', + '--completed-fence-commit', + 'b'.repeat(40), + '--completed-fence-operation', + 'operation-1', + '--completed-fence-state-serial', + '61', + '--completed-fence-plan-generation', + '123', + '--completed-fence-state-generation', + '456', + '--completed-fence-state-sha256', + 'c'.repeat(64), + '--fence-broker-service-account', + 'fence-broker@example.gserviceaccount.com' + ] + assert.equal( + parseMultiTargetArguments(args).completedFenceRecovery + .terraformStateSerial, + 61 + ) + assert.throws( + () => parseMultiTargetArguments(args.slice(0, -2)), + /recovery inputs are invalid/ + ) +}) + +test('requires the cutover source to remain existing-only', () => { + assert.throws( + () => + cutoverMembership(topology(), { + sourceCellId: 'source', + targetCellIds: ['target1'], + generalCellIds: ['source', 'target2'] + }), + /source must remain existing-only/ + ) +}) + +test('requires exact cutover connection-capacity evidence and live headroom', () => { + const status = { + draining: false, + connectionCapacity: { + hardCap: 600, + controlRebindReserve: 100, + ordinaryConnectionLimit: 500, + unobservedBound: 40, + normalAdmissionPause: 460, + pendingControlReservations: 20, + heartbeatFresh: true + }, + runtimeConnectionCapacity: { + hardCap: 600, + controlRebindReserve: 100, + ordinaryConnectionLimit: 500, + unobservedBound: 40, + normalAdmissionPause: 460 + }, + process: { + enforcedConnectionUnits: 400, + preAuthConnections: 3 + } + } + assert.doesNotThrow(() => assertCutoverCellReady('target1', status, 40)) + assert.throws( + () => + assertCutoverCellReady( + 'target1', + { + ...status, + runtimeConnectionCapacity: { + ...status.runtimeConnectionCapacity, + unobservedBound: 39 + } + }, + 40 + ), + /reviewed connection-capacity policy/ + ) + assert.throws( + () => + assertCutoverCellReady( + 'target1', + { + ...status, + connectionCapacity: { + ...status.connectionCapacity, + pendingControlReservations: 60 + } + }, + 40 + ), + /normal-admission connection headroom/ + ) + assert.throws( + () => + assertCutoverCellReady( + 'target1', + { + ...status, + process: { ...status.process, preAuthConnections: 45 } + }, + 40 + ), + /pre-auth connection headroom/ + ) +}) + +test('cuts over only after every proposed cell passes exact readiness evidence', async () => { + await withTopology(async (file) => { + const testHarness = harness() + await runMultiTargetDeployment( + config(file, 'cutover-admission'), + testHarness.overrides + ) + assert.deepEqual(testHarness.selector(), { + generation: 1, + attemptId: testHarness.selector().attemptId, + membership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + assert.equal(testHarness.events.at(-1).event, 'admission_selector_cutover') + assert.deepEqual(Object.fromEntries(testHarness.runtimeInspections), { + target1: 2, + target2: 2, + general: 2 + }) + }) +}) + +test('rejects a different active selector before readiness checks or revision mutation', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + selectorMembership: { + existingOnly: ['source', 'target2'], + migrationOnly: ['target1'], + general: ['general'] + } + }) + await assert.rejects( + runMultiTargetDeployment( + config(file, 'cutover-admission'), + testHarness.overrides + ), + /already active with different membership/ + ) + assert.equal(testHarness.runtimeInspections.size, 0) + }) +}) + +test('registers new Terraform targets as one selector generation', async () => { + await withTopology(async (file) => { + const reviewed = topology() + reviewed.target1.connection_hard_cap = 1_000 + reviewed.target1.connection_unobserved_bound = 60 + writeFileSync(file, JSON.stringify(reviewed)) + const testHarness = harness({ + selectorMembership: { + existingOnly: ['source'], + migrationOnly: [], + general: ['general'] + }, + activeDirectorMinimum: 5 + }) + await runMultiTargetDeployment( + { ...config(file, 'add-migration-cells'), directorMinimumInstances: 5 }, + testHarness.overrides + ) + assert.deepEqual(testHarness.selector(), { + generation: 2, + attemptId: 'add_cells_test', + membership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + assert.equal(testHarness.events.at(-1).event, 'migration_cells_added') + assert.deepEqual(testHarness.addedCells(), [ + { + cellId: 'target1', + cellUrl: topology().target1.origin, + capacityRequests: 4_000, + region: 'us-central1', + connectionHardCap: 1_000, + connectionUnobservedBound: 60 + }, + { + cellId: 'target2', + cellUrl: topology().target2.origin, + capacityRequests: 4_000, + region: 'us-central1', + connectionHardCap: 600, + connectionUnobservedBound: 40 + } + ]) + assert.equal(testHarness.runtimeInspections.size, 0) + }) +}) + +test('promotes exactly one healthy migration cell to general admission', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'promote-general-cell'), + testHarness.overrides + ) + assert.deepEqual(testHarness.selector(), { + generation: 2, + attemptId: 'promote_cell_test', + membership: { + existingOnly: ['source'], + migrationOnly: ['target2'], + general: ['general', 'target1'] + } + }) + assert.equal(testHarness.events.at(-1).event, 'migration_cell_promoted_general') + }) +}) + +test('rejects promotion below the configured director floor', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await assert.rejects( + runMultiTargetDeployment( + { ...config(file, 'promote-general-cell'), directorMinimumInstances: 2 }, + testHarness.overrides + ), + /active director is not compatible/ + ) + assert.equal(testHarness.events.length, 0) + }) +}) + +test('rejects general promotion unless the exact cell is migration-only', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target2'], + general: ['general', 'target1'] + } + }) + await assert.rejects( + runMultiTargetDeployment( + config(file, 'promote-general-cell'), + testHarness.overrides + ), + /must be migration-only/ + ) + assert.equal(testHarness.events.length, 0) + }) +}) + +test('retires exactly one migration cell without changing other membership', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'retire-migration-cell'), + testHarness.overrides + ) + assert.deepEqual(testHarness.selector(), { + generation: 2, + attemptId: 'retire_cell_test', + membership: { + existingOnly: ['source', 'target1'], + migrationOnly: ['target2'], + general: ['general'] + } + }) + assert.equal(testHarness.events.at(-1).event, 'migration_cell_retired') + assert.equal(testHarness.runtimeInspections.size, 0) + }) +}) + +test('rejects retirement unless the exact cell is migration-only', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target2'], + general: ['general', 'target1'] + } + }) + await assert.rejects( + runMultiTargetDeployment( + config(file, 'retire-migration-cell'), + testHarness.overrides + ), + /must be migration-only/ + ) + assert.equal(testHarness.events.length, 0) + }) +}) + +test('rejects overlapping generations and allocates below the connection ceiling', () => { + const selected = selectMultiTargetDeployments(topology(), 'source', ['target1', 'target2']) + assert.equal(selected.targets.length, 2) + const planned = allocateTargetQuotas({ + sourceAssignments: 4, + sourceConnections: 5, + requiredTargetUnits: 8, + connectionCeiling: 6, + targets: [ + { + cellId: 'target1', + currentConnections: 0, + availableConnectionReservations: 10, + availableTargetUnits: 10 + }, + { + cellId: 'target2', + currentConnections: 0, + availableConnectionReservations: 10, + availableTargetUnits: 10 + } + ] + }) + assert.deepEqual( + planned.map(({ cellId, quota, projectedConnections }) => ({ + cellId, + quota, + projectedConnections + })), + [ + { cellId: 'target1', quota: 2, projectedConnections: 3 }, + { cellId: 'target2', quota: 2, projectedConnections: 3 } + ] + ) + assert.throws(() => + allocateTargetQuotas({ + sourceAssignments: 4, + sourceConnections: 20, + requiredTargetUnits: 8, + connectionCeiling: 4, + targets: [ + { + cellId: 'target1', + currentConnections: 0, + availableConnectionReservations: 10, + availableTargetUnits: 10 + }, + { + cellId: 'target2', + currentConnections: 0, + availableConnectionReservations: 10, + availableTargetUnits: 10 + } + ] + }) + ) + assert.deepEqual( + allocateTargetQuotas({ + sourceAssignments: 1, + sourceConnections: 700, + requiredTargetUnits: 1, + connectionCeiling: 1_000, + targets: [ + { + cellId: 'target-600', + currentConnections: 0, + connectionCeiling: 600, + availableConnectionReservations: 1, + availableTargetUnits: 1 + }, + { + cellId: 'target-1000', + currentConnections: 0, + connectionCeiling: 1_000, + availableConnectionReservations: 1, + availableTargetUnits: 1 + } + ] + }).map(({ cellId, projectedConnections }) => ({ cellId, projectedConnections })), + [ + { cellId: 'target-600', projectedConnections: 699 }, + { cellId: 'target-1000', projectedConnections: 700 } + ] + ) +}) + +test('assumes every unbound source connection can land on one target', () => { + const targets = [ + { + cellId: 'target1', + currentConnections: 0, + availableConnectionReservations: 10, + availableTargetUnits: 10 + }, + { + cellId: 'target2', + currentConnections: 0, + availableConnectionReservations: 10, + availableTargetUnits: 10 + } + ] + const planned = allocateTargetQuotas({ + sourceAssignments: 4, + sourceConnections: 9, + requiredTargetUnits: 8, + connectionCeiling: 8, + targets + }) + assert.deepEqual( + planned.map(({ quota, projectedConnections }) => ({ quota, projectedConnections })), + [ + { quota: 2, projectedConnections: 7 }, + { quota: 2, projectedConnections: 7 } + ] + ) + assert.throws(() => + allocateTargetQuotas({ + sourceAssignments: 4, + sourceConnections: 9, + requiredTargetUnits: 8, + connectionCeiling: 7, + targets + }) + ) +}) + +test('serializes deterministic target quotas and completes after drain acceptance', async () => { + await withTopology(async (file) => { + const testHarness = harness() + await runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides) + assert.deepEqual(testHarness.batches, [ + ['target1', 2], + ['target2', 2] + ]) + assert.deepEqual(testHarness.stateChanges.slice(0, 3), [ + ['source', false], + ['target1', true], + ['target2', true] + ]) + assert.equal(testHarness.events.some((event) => event.event === 'source_drain_accepted'), true) + assert.equal(testHarness.events.at(-1).event, 'multi_target_complete') + }) +}) + +test('uses fresh aggregate logs for a legacy source without runtime counts', async () => { + await withTopology(async (file) => { + const testHarness = harness({ legacySource: true }) + await runMultiTargetDeployment(config(file), testHarness.overrides) + assert.equal(testHarness.events.at(-1).sourceConnections, 5) + }) +}) + +test('rejects stale legacy source telemetry', async () => { + await withTopology(async (file) => { + const testHarness = harness({ legacySource: true, legacyMetricAgeMs: 90_001 }) + await assert.rejects( + runMultiTargetDeployment(config(file), testHarness.overrides), + /runtime metrics are stale/ + ) + }) +}) + +test('refuses a drain with an expiring lease and restores pre-drain admission', async () => { + await withTopology(async (file) => { + const testHarness = harness({ leaseRemainingMs: 599_999 }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), + /insufficient time/ + ) + assert.deepEqual(testHarness.stateChanges.slice(-3), [ + ['source', true], + ['target1', false], + ['target2', false] + ]) + assert.equal( + testHarness.events.some((event) => event.event === 'source_drain_accepted'), + false + ) + }) +}) + +test('preserves forward recovery when target registration status is unavailable', async () => { + await withTopology(async (file) => { + const testHarness = harness({ failEvacuationStatus: true }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), + /cannot prove zero target registrations/ + ) + assert.equal( + testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), + false + ) + assert.equal( + testHarness.stateChanges.some( + ([cellId, enabled]) => cellId.startsWith('target') && !enabled + ), + false + ) + assert.deepEqual(testHarness.events.at(-1), { + event: 'multi_forward_recovery_required', + targetRegistered: null, + reason: 'registration_status_unavailable' + }) + }) +}) + +test('audits partial migration state without requiring new migration headroom', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + capacitySourceAssignments: [1_000], + capacityRequiredTargetUnits: [2_000] + }) + await runMultiTargetDeployment(config(file, 'audit'), testHarness.overrides) + assert.equal(testHarness.capacityReads(), 0) + assert.deepEqual(testHarness.events.at(-1), { + event: 'multi_target_audit', + inProgress: 4, + targetRegistered: 0, + registeredSourceActive: 0, + registeredCompletable: 0, + registeredTargetInactive: 0, + completed: 0, + blocked: 0, + expiredUnregistered: 0, + repairableExpiredUnregistered: 0, + abortableExpiredUnregistered: 0, + blockedExpiredUnregistered: 0, + blockedExpiredOnNewerTargetAssignment: 0 + }) + }) +}) + +test('never restores source admission after drain acceptance', async () => { + await withTopology(async (file) => { + const testHarness = harness({ failAfterDrain: true }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), + /injected_completion_failure/ + ) + assert.equal(testHarness.events.some((event) => event.event === 'source_drain_accepted'), true) + assert.equal( + testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), + false + ) + assert.equal(testHarness.events.at(-1).event, 'multi_forward_recovery_required') + }) +}) + +test('never restores source admission after the send transition becomes ambiguous', async () => { + await withTopology(async (file) => { + const testHarness = harness({ loseDrainSendResponse: true }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), + /injected_send_transition_response_loss/ + ) + assert.equal( + testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), + false + ) + assert.equal(testHarness.events.at(-1).event, 'multi_forward_recovery_required') + }) +}) + +test('freezes forward recovery when a legacy drain response is ambiguous', async () => { + await withTopology(async (file) => { + const testHarness = harness({ loseDrainBeforeAccept: true }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), + /injected_drain_response_loss/ + ) + assert.equal( + testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), + false + ) + await assert.rejects( + runMultiTargetDeployment(config(file, 'recover-forward'), testHarness.overrides), + /drain_application_receipt_missing/ + ) + assert.deepEqual(testHarness.drainGraces, [120_000]) + }) +}) + +test('resumes an exact prepared drain without allocating new migrations', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + preparedDrainAttempt: true, + preexistingRegisteredMigrations: 2, + sourceAssignments: 0, + sourceRequiredTargetUnits: 0, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.deepEqual(testHarness.drainGraces, [120_000]) + assert.deepEqual(testHarness.batches, []) + assert.equal(testHarness.capacityReads(), 8) + assert.equal( + testHarness.events.some( + (event) => event.event === 'source_prepared_drain_recovered' + ), + true + ) + assert.deepEqual(testHarness.events.at(-1), { + event: 'multi_target_complete', + sourceCellId: 'source' + }) + }) +}) + +test('registers only remaining assignments before recovering a replacement drain', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + preexistingRegisteredMigrations: 1, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.deepEqual(testHarness.batches, [ + ['target1', 2], + ['target2', 2] + ]) + assert.equal(testHarness.capacityReads(), 8) + assert.deepEqual(testHarness.drainGraces, [0]) + const eventNames = testHarness.events.map((event) => event.event) + assert.equal( + testHarness.events.find( + (event) => event.event === 'multi_forward_recovery_preflight' + ).targetRegistered, + 2 + ) + assert.ok( + eventNames.lastIndexOf('multi_migration_batch') < + eventNames.indexOf('multi_forward_recovery_ready_to_drain') + ) + assert.ok( + eventNames.indexOf('multi_forward_recovery_ready_to_drain') < + eventNames.indexOf('source_recovery_drain_accepted') + ) + }) +}) + +test('refreshes registered migration leases before the recovery drain gate', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + preexistingRegisteredMigrations: 2, + sourceAssignments: 0, + sourceRequiredTargetUnits: 0, + leaseRemainingMs: 599_999, + refreshedLeaseRemainingMs: 900_000, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.deepEqual(testHarness.drainGraces, [0]) + }) +}) + +test('waits for catch-up migrations to gain durable target ownership', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + recoveryRegistrationDelayReads: 2, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + const ownershipEvents = testHarness.events.filter( + (event) => event.event === 'multi_recovery_target_ownership' + ) + assert.equal(ownershipEvents.length, 3) + assert.equal(ownershipEvents[0].inProgress > ownershipEvents[0].targetRegistered, true) + assert.equal(ownershipEvents.at(-1).inProgress, ownershipEvents.at(-1).targetRegistered) + assert.deepEqual(testHarness.drainGraces, [0]) + }) +}) + +test('times out before drain when catch-up migrations remain unregistered', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + let now = 0 + testHarness.overrides.now = () => now + testHarness.overrides.wait = async (ms) => { + now += ms + } + await assert.rejects( + runMultiTargetDeployment( + { ...config(file, 'recover-forward'), timeoutMs: 3 }, + testHarness.overrides + ), + /timed out waiting for recovery target ownership/ + ) + assert.deepEqual(testHarness.drainGraces, []) + }) +}) + +test('drains a fully unregistered recovery within the reviewed bound', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + { + ...config(file, 'recover-forward'), + unobservedConnectionBound: 4 + }, + testHarness.overrides + ) + assert.deepEqual(testHarness.drainGraces, [0]) + assert.equal( + testHarness.events.some( + (event) => event.event === 'multi_recovery_bounded_unregistered' + ), + true + ) + }) +}) + +test('drains mixed target registrations within the unobserved bound', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + preexistingRegisteredMigrations: 1, + recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + { + ...config(file, 'recover-forward'), + unobservedConnectionBound: 2 + }, + testHarness.overrides + ) + assert.deepEqual(testHarness.drainGraces, [0]) + const event = testHarness.events.find( + (entry) => entry.event === 'multi_recovery_bounded_mixed_registration' + ) + assert.equal(event.unregistered, 2) + assert.equal(event.targetRegistered, 2) + }) +}) + +test('reissues a proven non-delivered recovery drain and settles bounded offline clients', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + recoveryAlreadyAttempted: true, + preexistingRegisteredMigrations: 1, + unregisteredTargetMigrations: 1, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + { + ...config(file, 'recover-forward'), + unobservedConnectionBound: 2 + }, + testHarness.overrides + ) + assert.deepEqual(testHarness.drainGraces, [0]) + assert.equal( + testHarness.events.some( + (event) => event.event === 'source_recovery_drain_reissued_after_non_delivery' + ), + true + ) + assert.equal( + testHarness.events.some( + (event) => event.event === 'multi_migration_bounded_offline_complete' + ), + true + ) + }) +}) + +test('rejects mixed target registrations above the unobserved bound', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + preexistingRegisteredMigrations: 1, + recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + let now = 0 + testHarness.overrides.now = () => now + testHarness.overrides.wait = async (ms) => { + now += ms + } + await assert.rejects( + runMultiTargetDeployment( + { + ...config(file, 'recover-forward'), + timeoutMs: 3, + unobservedConnectionBound: 1 + }, + testHarness.overrides + ), + /timed out waiting for recovery target ownership/ + ) + assert.deepEqual(testHarness.drainGraces, []) + }) +}) + +test('times out before drain while a target registration remains source-active', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + preexistingRegisteredMigrations: 1, + registeredSourceActive: 1, + recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + let now = 0 + testHarness.overrides.now = () => now + testHarness.overrides.wait = async (ms) => { + now += ms + } + await assert.rejects( + runMultiTargetDeployment( + { + ...config(file, 'recover-forward'), + timeoutMs: 3, + unobservedConnectionBound: 2 + }, + testHarness.overrides + ), + /timed out waiting for recovery target ownership/ + ) + assert.deepEqual(testHarness.drainGraces, []) + }) +}) + +test('allows recovery drain with durable offline target registrations', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + offlineTargetMigrations: 1, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.deepEqual(testHarness.drainGraces, [0]) + }) +}) + +test('catches up source assignments that arrive during recovery publication', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + sourceAssignments: 5, + sourceRequiredTargetUnits: 10, + capacitySourceAssignments: [4, 4, 4, 4, 1, 1, 1, 1], + capacityRequiredTargetUnits: [8, 8, 8, 8, 2, 2, 2, 2], + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.equal( + testHarness.batches.reduce((total, [, limit]) => total + limit, 0), + 5 + ) + assert.equal( + testHarness.events.some( + (event) => event.event === 'multi_forward_recovery_catch_up' + ), + true + ) + assert.deepEqual( + testHarness.events + .filter((event) => event.event === 'multi_target_preflight') + .map((event) => ({ + sourceConnections: event.sourceConnections, + observedSourceConnections: event.observedSourceConnections + })), + [ + { sourceConnections: 5, observedSourceConnections: 5 }, + { sourceConnections: 1, observedSourceConnections: 5 } + ] + ) + assert.deepEqual(testHarness.drainGraces, [0]) + }) +}) + +test('replans recover-forward after transactional target headroom rejection', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + headroomFailureTarget: 'target1', + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.equal( + testHarness.events.some( + (event) => + event.event === 'multi_recovery_target_headroom_paused' && + event.targetCellId === 'target1' + ), + true + ) + assert.equal( + testHarness.events.some( + (event) => event.event === 'multi_forward_recovery_catch_up' + ), + true + ) + assert.deepEqual(testHarness.drainGraces, [0]) + }) +}) + +test('uses conservative changing capacity samples during recovery', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + sourceAssignments: 5, + sourceRequiredTargetUnits: 10, + capacitySourceAssignments: [5, 4, 5, 4], + capacityRequiredTargetUnits: [10, 8, 10, 8], + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.equal( + testHarness.batches.reduce((total, [, limit]) => total + limit, 0), + 5 + ) + assert.deepEqual(testHarness.drainGraces, [0]) + }) +}) + +test('requires every final recovery sample to prove zero source assignments', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + preexistingRegisteredMigrations: 2, + sourceAssignments: 1, + sourceRequiredTargetUnits: 2, + capacitySourceAssignments: [1, 1, 1, 1, 0, 0, 0, 1], + capacityRequiredTargetUnits: [2, 2, 2, 2, 0, 0, 0, 2], + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.equal( + testHarness.events.some( + (event) => event.event === 'multi_forward_recovery_catch_up' + ), + true + ) + assert.deepEqual(testHarness.drainGraces, [0]) + }) +}) + +test('stops after bounded recovery catch-up cannot quiesce the source', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + recoverableDrain: true, + sourceAssignments: 10, + sourceRequiredTargetUnits: 20, + capacitySourceAssignments: Array.from({ length: 40 }, () => 1), + capacityRequiredTargetUnits: Array.from({ length: 40 }, () => 2), + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await assert.rejects( + runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ), + /did not quiesce within bounded recovery catch-up/ + ) + assert.equal( + testHarness.batches.reduce((total, [, limit]) => total + limit, 0), + 5 + ) + assert.deepEqual(testHarness.drainGraces, []) + }) +}) + +test('settles a drained source while registered target users remain offline', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + allowPreFenceCompletion: true, + offlineTargetMigrations: 1, + selectorMembership: { + existingOnly: ['source'], + migrationOnly: ['target1', 'target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'recover-forward'), + testHarness.overrides + ) + assert.deepEqual(testHarness.drainGraces, []) + assert.deepEqual(testHarness.events.at(-1), { + event: 'multi_target_complete', + sourceCellId: 'source' + }) + }) +}) + +test('fences a quiescent disabled source and completes through stale-source checks', async () => { + await withTopology(async (file) => { + const testHarness = harness({ fence: true }) + const authorizationByOrigin = new Map() + const fetch = testHarness.overrides.fetch + testHarness.overrides.fetch = async (url, options) => { + const parsed = new URL(url) + if (parsed.pathname.startsWith('/v1/admin/')) { + authorizationByOrigin.set( + parsed.origin, + new Set([ + ...(authorizationByOrigin.get(parsed.origin) ?? []), + options?.headers?.authorization + ]) + ) + } + return await fetch(url, options) + } + await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) + assert.equal(testHarness.fencedCompletions(), 2) + assert.deepEqual( + authorizationByOrigin.get('https://relay.example.com'), + new Set(['Bearer aaa.bbb.ccc', 'Bearer ddd.eee.fff']) + ) + assert.equal(authorizationByOrigin.has('https://a.relay.example.com'), false) + assert.equal(authorizationByOrigin.has('https://b.relay.example.com'), false) + assert.equal(authorizationByOrigin.has('https://c.relay.example.com'), false) + assert.deepEqual(testHarness.events.at(-1), { + event: 'source_fenced', + sourceCellId: 'source', + targetSize: 0 + }) + }) +}) + +test('adopts an already-fenced source only through the legacy no-op path', async () => { + await withTopology(async (file) => { + const testHarness = harness({ fence: true, alreadyFencedSource: true }) + await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) + assert.equal(testHarness.fencedCompletions(), 2) + assert.deepEqual( + testHarness.events.find( + ({ event }) => event === 'terraform_cell_fence_legacy_adopted' + ), + { event: 'terraform_cell_fence_legacy_adopted', cellId: 'source' } + ) + assert.deepEqual(testHarness.events.at(-1), { + event: 'source_fenced', + sourceCellId: 'source', + targetSize: 0 + }) + }) +}) + +test('refuses to attest a fence when selected targets omit an outgoing migration', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + alreadyFencedSource: true, + sourceOutgoingMigrations: 5 + }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /full migration coverage/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }) +}) + +test('clears the complete 156-row legacy backlog before reporting the source fenced', async () => { + const rollout = topology() + const additionalTarget = (id, hostname) => ({ + ...rollout.target2, + origin: `https://${hostname}.relay.example.com`, + zone: `us-central1-${hostname}`, + mig_name: `relay-${hostname}`, + instance_group: `https://compute.example/instanceGroups/relay-${hostname}`, + backend_name: `relay-${hostname}`, + backend_id: `https://compute.example/backendServices/relay-${hostname}`, + generation_identity: `https://compute.example/instanceTemplates/relay-${hostname}-abc`, + image: `us-central1-docker.pkg.dev/project/repo/relay@${digest(id)}` + }) + rollout.target3 = additionalTarget('e', 'e') + rollout.target4 = additionalTarget('f', 'f') + rollout.target5 = additionalTarget('1', 'g') + rollout.target6 = additionalTarget('2', 'h') + const migrationCounts = { + target1: 58, + target2: 63, + target3: 24, + target4: 10, + target5: 1, + target6: 0 + } + + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + alreadyFencedSource: true, + offlineTargetMigrationsByTarget: migrationCounts, + topologyValue: rollout + }) + const fenceConfig = config(file, 'fence-source') + fenceConfig.targetCellIds = Object.keys(migrationCounts) + + await runMultiTargetDeployment(fenceConfig, testHarness.overrides) + + const sourceFencedIndex = testHarness.timeline.findIndex( + ({ kind, event }) => kind === 'event' && event.event === 'source_fenced' + ) + assert.notEqual(sourceFencedIndex, -1) + let observedInitialTotal = 0 + for (const [targetCellId, initialCount] of Object.entries(migrationCounts)) { + const initialIndex = testHarness.timeline.findIndex( + (entry) => + entry.kind === 'evacuation_status' && + entry.targetCellId === targetCellId && + entry.inProgress === initialCount && + entry.registeredTargetInactive === initialCount + ) + const clearedIndex = testHarness.timeline.findIndex( + (entry) => + entry.kind === 'evacuation_status' && + entry.targetCellId === targetCellId && + entry.inProgress === 0 + ) + assert.notEqual(initialIndex, -1) + observedInitialTotal += testHarness.timeline[initialIndex].inProgress + if (initialCount > 0) assert.ok(clearedIndex > initialIndex) + else assert.ok(clearedIndex >= initialIndex) + assert.ok(clearedIndex < sourceFencedIndex) + } + assert.equal(observedInitialTotal, 156) + assert.equal(testHarness.fencedCompletions(), 6) + }, rollout) +}) + +test('refuses legacy adoption when the live MIG template changed', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + alreadyFencedSource: true, + liveMigTemplateOverrides: { + source: `${topology().source.generation_identity}-other` + } + }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /source MIG fence topology is unsafe/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }) +}) + +test('fences with reviewed 600 templates during the declared 1000 rollout', async () => { + const rollout = topology() + for (const [cellId, cell] of Object.entries(rollout)) { + cell.connection_unobserved_bound = 60 + if (['target1', 'target2'].includes(cellId)) cell.connection_hard_cap = 1_000 + } + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + templateUnobservedBound: 60, + directorUnobservedBound: 60 + }) + const fenceConfig = config(file, 'fence-source') + fenceConfig.unobservedConnectionBound = 60 + await runMultiTargetDeployment(fenceConfig, testHarness.overrides) + assert.equal(testHarness.fencedCompletions(), 2) + }, rollout) +}) + +test('fences exact configured capacity when the saved Terraform output is legacy', async () => { + const rollout = topology() + for (const cell of Object.values(rollout)) { + delete cell.connection_hard_cap + delete cell.connection_unobserved_bound + } + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + templateUnobservedBound: 60, + directorUnobservedBound: 60 + }) + const fenceConfig = config(file, 'fence-source') + fenceConfig.unobservedConnectionBound = 60 + await runMultiTargetDeployment(fenceConfig, testHarness.overrides) + assert.equal(testHarness.fencedCompletions(), 2) + }, rollout) +}) + +test('refuses legacy Terraform output when live capacity differs from broker config', async () => { + const rollout = topology() + for (const cell of Object.values(rollout)) { + delete cell.connection_hard_cap + delete cell.connection_unobserved_bound + } + await withTopology(async (file) => { + const testHarness = harness({ fence: true }) + const fenceConfig = config(file, 'fence-source') + fenceConfig.unobservedConnectionBound = 60 + await assert.rejects( + runMultiTargetDeployment(fenceConfig, testHarness.overrides), + /instance template capacity differs from Terraform/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }, rollout) +}) + +test('refuses explicit null capacity as a legacy Terraform output', async () => { + const rollout = topology() + for (const cell of Object.values(rollout)) { + cell.connection_hard_cap = null + cell.connection_unobserved_bound = null + } + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + templateUnobservedBound: 60, + directorUnobservedBound: 60 + }) + const fenceConfig = config(file, 'fence-source') + fenceConfig.unobservedConnectionBound = 60 + await assert.rejects( + runMultiTargetDeployment(fenceConfig, testHarness.overrides), + /instance template capacity differs from Terraform/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }, rollout) +}) + +test('refuses a mixed legacy and declared capacity topology', async () => { + const rollout = topology() + for (const cell of Object.values(rollout)) { + delete cell.connection_hard_cap + delete cell.connection_unobserved_bound + } + rollout.general.connection_hard_cap = null + rollout.general.connection_unobserved_bound = null + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + templateUnobservedBound: 60, + directorUnobservedBound: 60 + }) + const fenceConfig = config(file, 'fence-source') + fenceConfig.unobservedConnectionBound = 60 + await assert.rejects( + runMultiTargetDeployment(fenceConfig, testHarness.overrides), + /instance template capacity differs from Terraform/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }, rollout) +}) + +test('fences after the active template reaches the declared 1000 capacity', async () => { + const rollout = topology() + for (const cellId of Object.keys(rollout)) { + rollout[cellId].connection_hard_cap = 1_000 + } + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + templateHardCap: 1_000, + directorHardCap: 1_000 + }) + await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) + assert.equal(testHarness.fencedCompletions(), 2) + }, rollout) +}) + +test('refuses a rollout predecessor whose template has the wrong bound', async () => { + const rollout = topology() + for (const cellId of ['target1', 'target2']) { + rollout[cellId].connection_hard_cap = 1_000 + rollout[cellId].connection_unobserved_bound = 60 + } + await withTopology(async (file) => { + const testHarness = harness({ fence: true }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /instance template capacity is outside reviewed rollout/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }, rollout) +}) + +test('refuses a rollout predecessor when the director reports another cap', async () => { + const rollout = topology() + for (const cellId of ['target1', 'target2']) { + rollout[cellId].connection_hard_cap = 1_000 + } + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + directorCapacityOverrides: { + target1: { hardCap: 1_000, unobservedBound: 40 } + } + }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /connection capacity differs from Terraform/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }, rollout) +}) + +test('preserves direct target runtime reads outside fence mode', async () => { + await withTopology(async (file) => { + const testHarness = harness({}) + const fetch = testHarness.overrides.fetch + const authorizationByOrigin = new Map() + testHarness.overrides.fetch = async (url, options) => { + const parsed = new URL(url) + if (parsed.pathname.startsWith('/v1/admin/')) { + authorizationByOrigin.set( + parsed.origin, + new Set([ + ...(authorizationByOrigin.get(parsed.origin) ?? []), + options?.headers?.authorization + ]) + ) + } + return await fetch(url, options) + } + + await runMultiTargetDeployment(config(file, 'preflight'), testHarness.overrides) + assert.deepEqual( + authorizationByOrigin.get('https://relay.example.com'), + new Set(['Bearer aaa.bbb.ccc']) + ) + assert.deepEqual( + authorizationByOrigin.get('https://b.relay.example.com'), + new Set(['Bearer aaa.bbb.ccc']) + ) + assert.deepEqual( + authorizationByOrigin.get('https://c.relay.example.com'), + new Set(['Bearer aaa.bbb.ccc']) + ) + }) +}) + +test('refuses to fence when a cell hostname routes to the wrong backend', async () => { + await withTopology(async (file) => { + const testHarness = harness({ fence: true, misroutedCell: 'target1' }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /retained route topology mismatch/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }) +}) + +test('refuses to fence when a cell route overrides the reviewed backend', async () => { + await withTopology(async (file) => { + const testHarness = harness({ fence: true, routeOverrideCell: 'target1' }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /retained route topology mismatch/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }) +}) + +test('refuses to fence when the live HTTPS frontend uses another URL map', async () => { + await withTopology(async (file) => { + const testHarness = harness({ fence: true, frontendMisbound: true }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /live frontend topology mismatch/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }) +}) + +test('refuses to fence when the fresh source heartbeat reports active work', async () => { + await withTopology(async (file) => { + const testHarness = harness({ fence: true, sourceObservedRequests: 1 }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /source fencing requires zero source-owned work/ + ) + assert.equal(testHarness.fencedCompletions(), 0) + }) +}) + +test('fences a quiescent source while registered target users remain offline', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + alreadyFencedSource: true, + offlineTargetMigrations: 1 + }) + await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) + assert.equal(testHarness.fencedCompletions(), 2) + assert.deepEqual(testHarness.events.at(-1), { + event: 'source_fenced', + sourceCellId: 'source', + targetSize: 0 + }) + }) +}) + +test('refuses to fence source-active or unregistered migrations', async () => { + for (const migrationState of [ + { registeredSourceActive: 1 }, + { unregisteredTargetMigrations: 1 } + ]) { + await withTopology(async (file) => { + const testHarness = harness({ fence: true, ...migrationState }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /full migration coverage and durable target ownership/ + ) + }) + } +}) + +test('resumes fenced completion after losing the resize response', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + fence: true, + loseResizeResponse: true, + resumeNeedsPreApply: true + }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), + /injected_resize_response_loss/ + ) + await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) + assert.equal(testHarness.fencedCompletions(), 2) + assert.deepEqual(testHarness.events.at(-1), { + event: 'source_fenced', + sourceCellId: 'source', + targetSize: 0 + }) + }) +}) + +test('records a proven pre-apply Terraform fence abort', async () => { + await withTopology(async (file) => { + const testHarness = harness({ existingFenceAttempt: true }) + await runMultiTargetDeployment( + config(file, 'abort-fence-source'), + testHarness.overrides + ) + assert.deepEqual(testHarness.events.at(-1), { + event: 'terraform_fence_aborted_before_apply', + cellId: 'source' + }) + }) +}) + +test('fences a failed registered target before aggregate supersession', async () => { + await withTopology(async (file) => { + const testHarness = harness({ supersede: true }) + await runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ) + assert.equal( + testHarness.stateChanges.some( + ([cellId, enabled]) => cellId === 'target2' && enabled + ), + true + ) + assert.deepEqual(testHarness.events.at(-1), { + event: 'registered_target_superseded', + sourceCellId: 'source', + failedTargetCellId: 'target1', + replacementTargetCellId: 'target2', + superseded: 2, + remainingUnregistered: 0 + }) + assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) + }) +}) + +test('fails closed when cleanup wins before aggregate supersession selects rows', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + supersede: true, + cleanupBeforeSupersession: true + }) + + await assert.rejects( + runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), + /registered target supersession count changed/ + ) + assert.deepEqual(testHarness.events, []) + }) +}) + +test('does not accept a capacity predecessor for target supersession', async () => { + const rollout = topology() + rollout.target2.connection_hard_cap = 1_000 + await withTopology(async (file) => { + const testHarness = harness({ supersede: true }) + await assert.rejects( + runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ), + /instance template capacity is outside reviewed rollout/ + ) + }, rollout) +}) + +test('does not accept capacity-bearing templates from legacy output for supersession', async () => { + const rollout = topology() + for (const cell of Object.values(rollout)) { + delete cell.connection_hard_cap + delete cell.connection_unobserved_bound + } + await withTopology(async (file) => { + const testHarness = harness({ supersede: true }) + await assert.rejects( + runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ), + /instance template capacity differs from Terraform/ + ) + }, rollout) +}) + +test('replaces a superseded unuploaded fence attempt before target fencing', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + supersede: true, + supersededFenceAttempt: true + }) + await runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ) + assert.deepEqual(testHarness.events[0], { + event: 'terraform_fence_superseded_before_upload', + cellId: 'target1', + previousFenceCommit: 'b'.repeat(40), + fenceCommit: 'a'.repeat(40) + }) + assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') + }) +}) + +test('recovers a pinned completed older fence before registered supersession', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + supersede: true, + completedFenceAttempt: true + }) + const deploymentConfig = config(file, 'supersede-target') + deploymentConfig.completedFenceRecovery = { + attemptId: '44444444-4444-4444-8444-444444444444', + fenceCommit: 'b'.repeat(40), + gceOperation: 'operation-1', + terraformStateSerial: 7, + planObjectGeneration: '123456789', + terraformStateObjectGeneration: '222222222', + terraformStateObjectSha256: 'c'.repeat(64), + principalEmail: 'fence-broker@example.gserviceaccount.com' + } + + await runMultiTargetDeployment(deploymentConfig, testHarness.overrides) + + assert.equal( + testHarness.events[0].event, + 'terraform_cell_fence_completed_attempt_recovered' + ) + assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') + }) +}) + +test('fails supersession on stale replacement heartbeat without calling its admin API', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + supersede: true, + replacementHeartbeatFresh: false + }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), + /no fresh matching director runtime snapshot/ + ) + assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) + assert.deepEqual(testHarness.events, []) + }) +}) + +test('fails supersession on mismatched director runtime without calling cell admin', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + supersede: true, + replacementRuntimeCellUrl: 'https://wrong.relay.example.com' + }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), + /no fresh matching director runtime snapshot/ + ) + assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) + assert.deepEqual(testHarness.events, []) + }) +}) + +test('reads the broker mutation token without treating the audience as the environment', async () => { + await withTopology(async (file) => { + const testHarness = harness({ supersede: true }) + delete testHarness.overrides.mutationIdentityToken + const previous = process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN + process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = 'ddd.eee.fff' + try { + await runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ) + } finally { + if (previous === undefined) { + delete process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN + } else { + process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = previous + } + } + assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') + }) +}) + +test('fails closed when the broker child has no mutation token', async () => { + await withTopology(async (file) => { + const testHarness = harness({ supersede: true }) + delete testHarness.overrides.mutationIdentityToken + const previous = process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN + delete process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN + try { + await assert.rejects( + runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ), + /requires a broker mutation identity token/ + ) + } finally { + if (previous !== undefined) { + process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = previous + } + } + assert.equal(testHarness.events.length, 0) + }) +}) + +test('uses Terraform fencing for selector-era target supersession', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + supersede: true, + selectorMembership: { + existingOnly: ['source', 'target1'], + migrationOnly: ['target2'], + general: ['general'] + } + }) + await runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ) + assert.deepEqual(testHarness.stateChanges, []) + assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') + }) +}) + +test('resumes failed-target supersession after losing the resize response', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + supersede: true, + loseResizeResponse: true + }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), + /injected_resize_response_loss/ + ) + await runMultiTargetDeployment( + config(file, 'supersede-target'), + testHarness.overrides + ) + assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') + }) +}) + +test('refuses to fence a failed target while its admission remains enabled', async () => { + await withTopology(async (file) => { + const testHarness = harness({ supersede: true, failedTargetEnabled: true }) + await assert.rejects( + runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), + /failed target admission must be disabled/ + ) + assert.equal(testHarness.events.length, 0) + }) +}) + +test('retries until two complete target-capacity rounds agree', async () => { + await withTopology(async (file) => { + const testHarness = harness({ capacitySourceAssignments: [4, 3] }) + await runMultiTargetDeployment(config(file), testHarness.overrides) + assert.equal(testHarness.capacityReads(), 8) + assert.equal(testHarness.events.at(-1).sourceAssignments, 4) + }) +}) + +test('fails closed after bounded inconsistent target-capacity rounds', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + capacitySourceAssignments: Array.from( + { length: 12 }, + (_, index) => index % 2 === 0 ? 4 : 3 + ) + }) + await assert.rejects( + runMultiTargetDeployment(config(file), testHarness.overrides), + /target capacity snapshots disagree/ + ) + assert.equal(testHarness.capacityReads(), 12) + assert.deepEqual(testHarness.stateChanges, []) + }) +}) + +test('emits aggregate capacity inputs before rejecting insufficient headroom', async () => { + await withTopology(async (file) => { + const testHarness = harness({ + capacityRequiredTargetUnits: Array.from({ length: 4 }, () => 20_000) + }) + await assert.rejects( + runMultiTargetDeployment(config(file), testHarness.overrides), + /multi-target connection or request-unit headroom exhausted/ + ) + const snapshot = testHarness.events.at(-1) + assert.equal(snapshot.event, 'multi_target_capacity_snapshot') + assert.equal(snapshot.sourceConnections, 5) + assert.equal(snapshot.sourceAssignments, 4) + assert.equal(snapshot.requiredTargetUnits, 20_000) + assert.deepEqual(snapshot.targets, [ + { + cellId: 'target1', + currentConnections: 0, + availableConnectionReservations: 460, + availableTargetUnits: 4_000 + }, + { + cellId: 'target2', + currentConnections: 0, + availableConnectionReservations: 460, + availableTargetUnits: 4_000 + } + ]) + }) +}) + +test('rejects quotas above enforced target reservation headroom', () => { + assert.throws( + () => + allocateTargetQuotas({ + sourceAssignments: 3, + sourceConnections: 3, + requiredTargetUnits: 3, + connectionCeiling: 600, + targets: [ + { + cellId: 'target1', + currentConnections: 0, + availableConnectionReservations: 1, + availableTargetUnits: 10 + }, + { + cellId: 'target2', + currentConnections: 0, + availableConnectionReservations: 1, + availableTargetUnits: 10 + } + ] + }), + /multi-target connection or request-unit headroom exhausted/ + ) +}) diff --git a/cloud/dev/scripts/github-smoke-token.mjs b/cloud/dev/scripts/github-smoke-token.mjs new file mode 100644 index 00000000000..f0bd67c1024 --- /dev/null +++ b/cloud/dev/scripts/github-smoke-token.mjs @@ -0,0 +1,133 @@ +const REQUEST_TIMEOUT_MS = 15_000 +const JWT_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/ + +export async function requestGitHubSmokeTokens( + authOrigin, + fetchImpl = fetch, + environment = process.env, + options = {} +) { + const origin = canonicalHttpsOrigin(authOrigin) + const requestUrl = environment.ACTIONS_ID_TOKEN_REQUEST_URL + const requestToken = environment.ACTIONS_ID_TOKEN_REQUEST_TOKEN + if (!requestUrl || !requestToken) throw new Error('GitHub OIDC request context is unavailable') + const audience = `${origin}/v1/internal/github-smoke-token` + const oidcUrl = new URL(requestUrl) + oidcUrl.searchParams.set('audience', audience) + const oidcResponse = await fetchImpl(oidcUrl, { + headers: { authorization: `Bearer ${requestToken}` }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }) + const oidc = await readJson(oidcResponse, 'GitHub OIDC request') + if (!oidcResponse.ok || !validJwt(oidc.value)) { + throw new Error(`GitHub OIDC request failed with ${oidcResponse.status}`) + } + const exchangeResponse = await fetchImpl(audience, { + method: 'POST', + headers: { + authorization: `Bearer ${oidc.value}`, + ...(options.relayAsiaLoad ? { 'content-type': 'application/json' } : {}) + }, + ...(options.relayAsiaLoad + ? { body: JSON.stringify({ relayAsiaLoad: parseRelayAsiaLoadOptions(options.relayAsiaLoad) }) } + : {}), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }) + const exchange = await readJson(exchangeResponse, 'Orca smoke identity exchange') + if (!exchangeResponse.ok) { + throw new Error(`Orca smoke identity exchange failed with ${exchangeResponse.status}`) + } + return { + ...parseAccessTokens(exchange.accessTokens), + ...(options.relayAsiaLoad + ? { + relayAsiaLoadPrincipals: parseRelayAsiaLoadPrincipals( + exchange.relayAsiaLoadPrincipals, + options.relayAsiaLoad.principalCount + ) + } + : {}) + } +} + +function parseRelayAsiaLoadOptions(value) { + if ( + !value || typeof value !== 'object' || + !Number.isSafeInteger(value.shardIndex) || value.shardIndex < 0 || value.shardIndex > 3 || + !Number.isSafeInteger(value.principalCount) || value.principalCount < 1 || + value.principalCount > 32 + ) throw new Error('Relay Asia load principal request is invalid') + return { v: 1, shardIndex: value.shardIndex, principalCount: value.principalCount } +} + +function canonicalHttpsOrigin(value) { + const url = new URL(value) + if (url.protocol !== 'https:' || url.pathname !== '/' || url.search || url.hash) { + throw new Error('auth origin must be canonical HTTPS') + } + return url.origin +} + +function validJwt(value) { + return typeof value === 'string' && value.length <= 8192 && JWT_PATTERN.test(value) +} + +function parseAccessTokens(value) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Orca smoke identity response is malformed') + } + const expected = ['outsider', 'owner', 'recipient'] + if (Object.keys(value).sort().join(',') !== expected.join(',')) { + throw new Error('Orca smoke identity response principals are invalid') + } + return Object.fromEntries( + expected.map((name) => { + const principal = value[name] + if ( + !principal || + typeof principal !== 'object' || + typeof principal.userId !== 'string' || + !/^[A-Za-z0-9_-]{1,128}$/.test(principal.userId) || + typeof principal.accessToken !== 'string' || + !validJwt(principal.accessToken) || + typeof principal.expiresAt !== 'number' || + principal.expiresAt <= Date.now() || + principal.expiresAt > Date.now() + 610_000 + ) { + throw new Error('Orca smoke identity response principal is malformed') + } + return [name, principal] + }) + ) +} + +function parseRelayAsiaLoadPrincipals(value, expectedCount) { + if (!Array.isArray(value) || value.length !== expectedCount) { + throw new Error('Relay Asia load principal response is invalid') + } + const userIds = new Set() + return value.map((principal, principalIndex) => { + if ( + !principal || typeof principal !== 'object' || + principal.principalIndex !== principalIndex || + typeof principal.userId !== 'string' || + !/^usr_relay_asia_load_[A-Za-z0-9_-]{32}$/.test(principal.userId) || + typeof principal.profileId !== 'string' || + principal.profileId !== principal.userId.replace(/^usr_/, 'prof_') || + userIds.has(principal.userId) || + typeof principal.accessToken !== 'string' || !validJwt(principal.accessToken) || + typeof principal.expiresAt !== 'number' || principal.expiresAt <= Date.now() || + principal.expiresAt > Date.now() + 610_000 + ) throw new Error('Relay Asia load principal response is malformed') + userIds.add(principal.userId) + return principal + }) +} + +async function readJson(response, label) { + try { + return await response.json() + } catch { + throw new Error(`${label} did not return JSON`) + } +} diff --git a/cloud/dev/scripts/github-smoke-token.test.mjs b/cloud/dev/scripts/github-smoke-token.test.mjs new file mode 100644 index 00000000000..9ffb0b864d8 --- /dev/null +++ b/cloud/dev/scripts/github-smoke-token.test.mjs @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { requestGitHubSmokeTokens } from './github-smoke-token.mjs' + +const jwt = (value) => `${value}.${value}.${value}` + +test('exchanges the runner OIDC token without returning request credentials', async () => { + const requests = [] + const fetchImpl = async (url, init) => { + requests.push({ url: String(url), init }) + if (requests.length === 1) return Response.json({ value: jwt('github') }) + return Response.json({ + accessTokens: Object.fromEntries( + ['owner', 'recipient', 'outsider'].map((name) => [ + name, + { userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 } + ]) + ) + }) + } + const result = await requestGitHubSmokeTokens( + 'https://auth-staging.onorca.dev', + fetchImpl, + { + ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token?api-version=1', + ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token' + } + ) + assert.equal(result.owner.userId, 'usr_owner') + assert.match(requests[0].url, /audience=https%3A%2F%2Fauth-staging\.onorca\.dev/) + assert.equal(requests[0].init.headers.authorization, 'Bearer runner-request-token') + assert.equal(requests[1].init.headers.authorization, `Bearer ${jwt('github')}`) +}) + +test('fails with bounded errors and never includes credentials', async () => { + await assert.rejects( + requestGitHubSmokeTokens( + 'https://auth-staging.onorca.dev', + async () => new Response('denied', { status: 403 }), + { + ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token', + ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'private-request-token' + } + ), + (error) => { + assert.doesNotMatch(String(error), /private-request-token|denied/) + return true + } + ) +}) + +test('requests and validates an exact Relay Asia principal batch', async () => { + const requests = [] + const principals = Array.from({ length: 32 }, (_, principalIndex) => { + const suffix = String(principalIndex).padStart(32, 'a') + return { + principalIndex, + userId: `usr_relay_asia_load_${suffix}`, + profileId: `prof_relay_asia_load_${suffix}`, + accessToken: jwt(`load${principalIndex}`), + expiresAt: Date.now() + 600_000 + } + }) + const result = await requestGitHubSmokeTokens( + 'https://auth-staging.onorca.dev', + async (url, init) => { + requests.push({ url: String(url), init }) + return requests.length === 1 + ? Response.json({ value: jwt('github') }) + : Response.json({ + accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [ + name, + { userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 } + ])), + relayAsiaLoadPrincipals: principals + }) + }, + { + ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token', + ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token' + }, + { relayAsiaLoad: { shardIndex: 3, principalCount: 32 } } + ) + assert.equal(result.relayAsiaLoadPrincipals.length, 32) + assert.deepEqual(JSON.parse(requests[1].init.body), { + relayAsiaLoad: { v: 1, shardIndex: 3, principalCount: 32 } + }) + assert.equal(requests[1].init.headers['content-type'], 'application/json') +}) + +test('rejects malformed or duplicate Relay Asia principal batches', async () => { + const environment = { + ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token', + ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token' + } + let request = 0 + await assert.rejects(requestGitHubSmokeTokens( + 'https://auth-staging.onorca.dev', + async () => ++request === 1 + ? Response.json({ value: jwt('github') }) + : Response.json({ + accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [ + name, + { userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 } + ])), + relayAsiaLoadPrincipals: [] + }), + environment, + { relayAsiaLoad: { shardIndex: 0, principalCount: 32 } } + ), /principal response is invalid/) +}) diff --git a/cloud/dev/scripts/infra.mjs b/cloud/dev/scripts/infra.mjs new file mode 100644 index 00000000000..6ab85df08b7 --- /dev/null +++ b/cloud/dev/scripts/infra.mjs @@ -0,0 +1,146 @@ +import { execFileSync } from 'node:child_process' +import { existsSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' +import { assertStagingRelayAwake } from './staging-relay-apply-guard.mjs' + +// The relay root keeps its historical path so every existing caller — 9 workflows, the fence +// broker, and the infra:* scripts — is unchanged when --root is omitted. The foundation and apps +// roots stay in the private repository with the services they own. +const ROOT_DIRECTORIES = { + relay: join('infra', 'terraform') +} + +const command = process.argv[2] +const environment = readEnvironment(process.argv.slice(3)) +const root = readRoot(process.argv.slice(3)) +const tool = process.env.IAC_TOOL || findTool() + +if (!command || !['init', 'plan', 'apply'].includes(command)) { + exitWithUsage() +} + +if (!environment) { + exitWithUsage('Missing --env staging|production') +} + +if (!root) { + exitWithUsage(`Unknown --root; expected one of ${Object.keys(ROOT_DIRECTORIES).join('|')}`) +} + +const rootDirectory = ROOT_DIRECTORIES[root] +const terraformDir = join(process.cwd(), rootDirectory) +const backendConfig = join(terraformDir, 'backend', `${environment}.hcl`) +const varFile = join(terraformDir, 'environments', `${environment}.tfvars`) + +if (!existsSync(backendConfig)) { + throw new Error(`Backend config not found: ${backendConfig}`) +} + +if (!existsSync(varFile)) { + throw new Error(`Variable file not found: ${varFile}`) +} + +const chdir = `-chdir=${rootDirectory}` + +if (command === 'init') { + run([chdir, 'init', `-backend-config=backend/${environment}.hcl`]) +} else if (command === 'plan') { + run([ + chdir, + 'plan', + `-var-file=environments/${environment}.tfvars`, + `-out=${environment}.tfplan` + ]) +} else { + // A normal staging apply must not implicitly wake or partially mutate a sleeping data plane. + // Only the relay root can touch that data plane; the guard would refuse app work for no reason. + if (environment === 'staging' && root === 'relay') assertStagingRelayAwake() + run([chdir, 'apply', `${environment}.tfplan`]) +} + +function readEnvironment(args) { + const envIndex = args.indexOf('--env') + if (envIndex >= 0) { + return args[envIndex + 1] + } + + return process.env.ORCA_CLOUD_ENV +} + +function readRoot(args) { + const rootIndex = args.indexOf('--root') + const requested = rootIndex >= 0 ? args[rootIndex + 1] : 'relay' + return requested in ROOT_DIRECTORIES ? requested : undefined +} + +function findTool() { + for (const candidate of ['tofu', 'terraform']) { + try { + execFileSync(candidate, ['version'], { stdio: 'ignore' }) + return candidate + } catch { + // Try the next compatible IaC binary. + } + } + + throw new Error('Install Terraform or OpenTofu, or set IAC_TOOL.') +} + +function run(args) { + execFileSync(tool, args, { env: terraformEnv(), stdio: 'inherit' }) +} + +function terraformEnv() { + if ( + process.env.GOOGLE_APPLICATION_CREDENTIALS || + process.env.GOOGLE_CREDENTIALS || + process.env.GOOGLE_OAUTH_ACCESS_TOKEN + ) { + return process.env + } + + const token = readGcloudAccessToken() + if (!token) { + return process.env + } + + // Local convenience: Terraform uses ADC, while engineers often only have + // gcloud CLI auth. CI should use Workload Identity instead. + return { ...process.env, GOOGLE_OAUTH_ACCESS_TOKEN: token } +} + +function readGcloudAccessToken() { + for (const candidate of gcloudCandidates()) { + try { + return execFileSync(candidate, ['auth', 'print-access-token'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'] + }).trim() + } catch { + // Try the next common gcloud location. + } + } + + return null +} + +function gcloudCandidates() { + return [ + process.env.GCLOUD_PATH, + 'gcloud', + join(homedir(), 'Downloads', 'google-cloud-sdk', 'bin', 'gcloud'), + join(homedir(), 'google-cloud-sdk', 'bin', 'gcloud') + ].filter(Boolean) +} + +function exitWithUsage(message) { + if (message) { + console.error(message) + } + + console.error( + 'Usage: pnpm infra: --env staging|production [--root relay]' + ) + process.exit(1) +} diff --git a/cloud/dev/scripts/infra.test.mjs b/cloud/dev/scripts/infra.test.mjs new file mode 100644 index 00000000000..267bcf540ac --- /dev/null +++ b/cloud/dev/scripts/infra.test.mjs @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' + +const repository = fileURLToPath(new URL('../../', import.meta.url)) +const script = 'dev/scripts/infra.mjs' + +// IAC_TOOL=echo prints the argv the real binary would have received, so the root a flag selects +// is observable without running Terraform. +function invoke(args) { + return execFileSync('node', [script, ...args], { + cwd: repository, + encoding: 'utf8', + env: { ...process.env, IAC_TOOL: 'echo' } + }).trim() +} + +function rejects(args) { + try { + execFileSync('node', [script, ...args], { + cwd: repository, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, IAC_TOOL: 'echo' } + }) + } catch (error) { + return error.stderr + } + throw new Error(`expected ${args.join(' ')} to exit non-zero`) +} + +// Why: 9 relay workflows, the fence broker, and the three infra:* package scripts all invoke this +// without --root. If the default ever moves off infra/terraform they break silently at the plan. +test('omitting --root keeps every existing caller on the relay root', () => { + for (const environment of ['staging', 'production']) { + assert.equal( + invoke(['init', '--env', environment]), + `-chdir=infra/terraform init -backend-config=backend/${environment}.hcl` + ) + assert.equal( + invoke(['plan', '--env', environment]), + `-chdir=infra/terraform plan -var-file=environments/${environment}.tfvars -out=${environment}.tfplan` + ) + } +}) + +// Only the relay root ships here; the foundation and apps roots stay in the private repository. +test('each root name selects exactly its own directory', () => { + const directories = { relay: 'infra/terraform' } + for (const [root, directory] of Object.entries(directories)) { + for (const environment of ['staging', 'production']) { + assert.equal( + invoke(['init', '--env', environment, '--root', root]), + `-chdir=${directory} init -backend-config=backend/${environment}.hcl` + ) + } + } +}) + +test('an unknown root fails closed rather than falling back to the relay root', () => { + const stderr = rejects(['plan', '--env', 'staging', '--root', 'releay']) + assert.match(stderr, /Unknown --root/) + assert.doesNotMatch(stderr, /infra\/terraform /) +}) + +test('a missing environment still fails before any root is resolved', () => { + assert.match(rejects(['plan']), /Missing --env/) +}) + +// Why: the guard refuses a staging apply while the relay data plane is asleep. Applying it to the +// app or foundation roots would block work that never touches that data plane. +test('the sleeping staging relay guard is scoped to the relay root', () => { + const source = readFileSync(new URL('./infra.mjs', import.meta.url), 'utf8') + assert.match(source, /environment === 'staging' && root === 'relay'/) +}) diff --git a/cloud/dev/scripts/load-relay-controls.mjs b/cloud/dev/scripts/load-relay-controls.mjs new file mode 100644 index 00000000000..32cd1858dd6 --- /dev/null +++ b/cloud/dev/scripts/load-relay-controls.mjs @@ -0,0 +1,570 @@ +import { createHash, createPrivateKey, createPublicKey } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { monitorEventLoopDelay } from 'node:perf_hooks' +import { setTimeout as delay } from 'node:timers/promises' +import { RelayLoadControlPeer } from './relay-load-control-peer.mjs' +import { requestGitHubSmokeTokens } from './github-smoke-token.mjs' +import { relayLoadFailureReason } from './relay-load-connection-failure.mjs' +import { + assertRelayLoadDirectorCapacityToken, + waitForRelayLoadDirectorCapacity, + waitForRelayLoadRequestUnits +} from './relay-load-director-capacity-gate.mjs' +import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs' +import { + proveRelayLoadPlacementBoundary, + proveRelayLoadRegionalFallback +} from './relay-load-placement-boundary.mjs' +import { + proveRelayLoadRebindBoundary, + waitForRelayLoadRebindGate +} from './relay-load-rebind-boundary.mjs' +import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs' +import { + openRelayLoadInviteOffers, + proveRelayLoadRequestUnitBoundary +} from './relay-load-request-unit-boundary.mjs' +import { + assertRelayLoadRampAccepted, + relayLoadRunHasDisallowedFailures, + runRelayLoadWithShutdown +} from './relay-load-run-lifecycle.mjs' +import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs' +import { + parseRelayLoadArguments, + relayLoadPrincipalIndex, + relayLoadReaderEvidenceError, + relayLoadSpliceIndexes, + relayLoadSpliceProfile, + relayLoadSpliceStartDelayMs +} from './relay-load-profile.mjs' + +function signingKey(path) { + if (!path) return {} + const key = createPrivateKey(readFileSync(path, 'utf8')) + const signingKeyId = createHash('sha256') + .update(createPublicKey(key).export({ type: 'spki', format: 'der' })) + .digest('base64url') + .slice(0, 16) + return { signingKey: key, signingKeyId } +} + +function report(state, final = false) { + const elapsedSeconds = Math.max(1, (Date.now() - state.startedAt) / 1000) + const memory = process.memoryUsage() + const cpu = process.cpuUsage(state.generatorBaselineCpu) + const rssMiB = memory.rss / 1_048_576 + state.generatorPeakRssMiB = Math.max(state.generatorPeakRssMiB, rssMiB) + const readerQueueEvidence = state.readerEvidence?.snapshot() ?? [] + const output = { + event: final ? 'relay_load_complete' : 'relay_load_progress', + controls: state.controls, + shardCount: state.shardCount, + shardIndex: state.shardIndex, + configuredRampSeconds: state.rampMs / 1000, + configuredSteadySeconds: state.durationMs / 1000, + configuredSpliceHoldSeconds: state.spliceHoldMs / 1000, + requiredLeaseHorizons: state.requiredLeaseHorizons, + configuredSplices: state.splices, + configuredSlowReaderSplices: state.slowReaderSplices, + configuredWedgedReaderSplices: state.wedgedReaderSplices, + active: state.active.size, + peakActive: state.peakActive, + steadyMinimumActive: state.steadyMinimumActive, + connected: state.connected, + connectionFailures: state.connectionFailures, + rampConnectionFailures: state.rampConnectionFailures, + steadyConnectionFailures: state.steadyConnectionFailures, + transitionConnectionFailures: state.transitionConnectionFailures, + connectionFailuresByReason: state.connectionFailuresByReason, + closes: state.closes, + unexpectedCloses: state.unexpectedCloses, + unexpectedClosesByCode: state.unexpectedClosesByCode, + drains: state.drains, + pings: state.pings, + pingRate: Number((state.pings / elapsedSeconds).toFixed(2)), + tokens: state.tokens, + tokenRate: Number((state.tokens / elapsedSeconds).toFixed(2)), + refreshes: state.refreshes, + refreshErrors: state.refreshErrors, + protocolErrors: state.protocolErrors, + socketErrors: state.socketErrors, + rebindProbesOpened: state.rebindProbesOpened, + rebindOverflowReason: state.rebindOverflowReason, + placementOverflowReason: state.placementOverflowReason, + regionalFallbacksProved: state.regionalFallbacksProved, + oldClientUsFirstProved: state.oldClientUsFirstProved, + stickyAssignmentProved: state.stickyAssignmentProved, + requestUnitInvitesOpened: state.requestUnitInvitesOpened, + requestUnitPrincipalCount: state.requestUnitPrincipalCount, + relayAsiaLoadPrincipalCount: state.relayAsiaLoadPrincipalCount, + requestUnitOverflowReason: state.requestUnitOverflowReason, + requestUnitCleanupProved: state.requestUnitCleanupProved, + phaseBarrierPassed: state.phaseBarrierPassed, + activeSplices: state.activeSplices, + peakActiveSplices: state.peakActiveSplices, + completedSplices: state.completedSplices, + failedSplices: state.failedSplices, + slowReaderSplicesCompleted: state.slowReaderSplicesCompleted, + wedgedReaderSplicesClosed: state.wedgedReaderSplicesClosed, + readerQueueEvidence, + readerQueuedBytesPeak: Math.max( + 0, + ...readerQueueEvidence.map(({ increaseBytes }) => increaseBytes) + ), + readerClosesByCode: state.readerClosesByCode, + controlHeadroom: Math.max(0, state.controls - state.active.size), + generatorRssMiB: Number(rssMiB.toFixed(1)), + generatorPeakRssMiB: Number(state.generatorPeakRssMiB.toFixed(1)), + generatorRssGrowthMiB: Number( + Math.max(0, state.generatorPeakRssMiB - state.generatorBaselineRssMiB).toFixed(1) + ), + generatorHeapUsedMiB: Number((memory.heapUsed / 1_048_576).toFixed(1)), + generatorCpuPercent: Number( + (((cpu.user + cpu.system) / 1_000_000 / elapsedSeconds) * 100).toFixed(1) + ), + generatorEventLoopP99Ms: Number((state.eventLoopDelay.percentile(99) / 1_000_000).toFixed(2)), + shutdownEvidence: final ? state.shutdownEvidence : undefined, + elapsedSeconds: Number(elapsedSeconds.toFixed(1)) + } + console.log(JSON.stringify(output)) + return output +} + +const config = parseRelayLoadArguments(process.argv.slice(2)) +let accessToken = process.env.ORCA_RELAY_LOAD_ACCESS_TOKEN +let accessTokenProviderForIndex +const adminToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN +if ( + config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0 || + config.slowReaderSplices + config.wedgedReaderSplices > 0 || + config.requestUnitOverflowProbes > 0 || config.requestUnitCleanupTimeoutMs > 0 +) { + assertRelayLoadDirectorCapacityToken({ + directorOrigin: config.directorOrigin, + adminToken + }, Date.now, + config.rampMs + config.durationMs + config.wedgedReaderHoldMs + + (config.phaseBarrierDir ? 2 * config.phaseBarrierTimeoutMs : 0) + + config.spliceRampMs + config.requestUnitCleanupTimeoutMs + 120_000) +} +const key = signingKey(config.signingKeyFile) +if (!accessToken && !key.signingKey && process.env.ACTIONS_ID_TOKEN_REQUEST_URL && + process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN) { + let tokens + let refresh + const loadOptions = config.relayAsiaLoadPrincipalCount > 0 + ? { + relayAsiaLoad: { + shardIndex: config.shardIndex, + principalCount: config.relayAsiaLoadPrincipalCount + } + } + : undefined + const smokeTokens = async () => { + const expiresAt = config.relayAsiaLoadPrincipalCount > 0 + ? tokens?.relayAsiaLoadPrincipals?.[0]?.expiresAt + : tokens?.owner?.expiresAt + if (expiresAt > Date.now() + 60_000) return tokens + refresh ??= requestGitHubSmokeTokens( + config.authOrigin, + fetch, + process.env, + loadOptions + ) + try { + tokens = await refresh + return tokens + } finally { + refresh = undefined + } + } + accessTokenProviderForIndex = (index) => async () => { + const current = await smokeTokens() + return config.relayAsiaLoadPrincipalCount > 0 + ? current.relayAsiaLoadPrincipals[ + relayLoadPrincipalIndex( + index, + config.shardCount, + current.relayAsiaLoadPrincipals.length + ) + ].accessToken + : current.owner.accessToken + } + await smokeTokens() +} +if (!accessToken && !accessTokenProviderForIndex && !key.signingKey) { + throw new Error('provide GitHub OIDC, ORCA_RELAY_LOAD_ACCESS_TOKEN, or --signing-key-file') +} +const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 }) +eventLoopDelay.enable() +const generatorBaselineRssMiB = process.memoryUsage().rss / 1_048_576 +const generatorBaselineCpu = process.cpuUsage() +const state = { + ...config, + startedAt: Date.now(), + active: new Set(), + peakActive: 0, + steadyMinimumActive: null, + steadyStarted: false, + connected: 0, + connectionFailures: 0, + rampConnectionFailures: 0, + steadyConnectionFailures: 0, + transitionConnectionFailures: 0, + connectionFailuresByReason: {}, + closes: 0, + unexpectedCloses: 0, + unexpectedClosesByCode: {}, + drains: 0, + pings: 0, + tokens: 0, + refreshes: 0, + refreshErrors: 0, + protocolErrors: 0, + socketErrors: 0, + rebindProbesOpened: 0, + rebindOverflowReason: null, + placementOverflowReason: null, + regionalFallbacksProved: 0, + oldClientUsFirstProved: 0, + stickyAssignmentProved: 0, + requestUnitInvitesOpened: 0, + requestUnitOverflowReason: null, + requestUnitCleanupProved: 0, + phaseBarrierPassed: false, + activeSplices: 0, + peakActiveSplices: 0, + completedSplices: 0, + failedSplices: 0, + slowReaderSplicesCompleted: 0, + wedgedReaderSplicesClosed: 0, + readerEvidence: null, + readerClosesByCode: {}, + generatorBaselineRssMiB, + generatorBaselineCpu, + generatorPeakRssMiB: generatorBaselineRssMiB, + peerShutdowns: 0, + shutdownEvidence: null, + eventLoopDelay, + stopping: false, + transitionWindow: false +} +const peers = new Map() +const reconnectTimers = new Set() + +async function readRuntimeQueuedBytes(origin) { + const response = await fetch(`${origin}/v1/admin/runtime-status`, { + method: 'POST', + headers: { authorization: `Bearer ${adminToken}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1 }), + signal: AbortSignal.timeout(5_000) + }) + if (response.status === 401 || response.status === 403) { + throw new Error('reader evidence identity was rejected') + } + if (!response.ok) throw new Error(`reader runtime status returned ${response.status}`) + const status = await response.json() + const queuedBytes = status?.runtime?.queuedBytes + if (!Number.isSafeInteger(queuedBytes) || queuedBytes < 0) { + throw new Error('reader runtime queued bytes are invalid') + } + return queuedBytes +} + +async function observeReaderPressure(input) { + if (!state.readerEvidence) throw new Error('reader evidence baseline is unavailable') + await state.readerEvidence.observe(input) + state.generatorPeakRssMiB = Math.max( + state.generatorPeakRssMiB, + process.memoryUsage().rss / 1_048_576 + ) +} + +function recordSteadyMinimum() { + if (!state.steadyStarted || state.stopping) return + state.steadyMinimumActive = Math.min(state.steadyMinimumActive, state.active.size) +} + +function scheduleReconnect(peer) { + if (state.stopping) return + const timeout = setTimeout(() => { + reconnectTimers.delete(timeout) + void connect(peer) + }, Math.floor(Math.random() * (config.reconnectMaxMs + 1))) + reconnectTimers.add(timeout) +} + +function observe(type, detail) { + if (type === 'connected') { + state.active.add(detail.index) + state.connected++ + state.peakActive = Math.max(state.peakActive, state.active.size) + recordSteadyMinimum() + } else if (type === 'closed') { + state.active.delete(detail.index) + state.closes++ + if (!detail.stopped && !detail.expectedDrain) { + state.unexpectedCloses++ + const code = String(detail.code) + state.unexpectedClosesByCode[code] = (state.unexpectedClosesByCode[code] ?? 0) + 1 + } + if (!detail.stopped) scheduleReconnect(peers.get(detail.index)) + recordSteadyMinimum() + } else if (type === 'drain') state.drains++ + else if (type === 'ping') state.pings++ + else if (type === 'token') state.tokens++ + else if (type === 'refresh') state.refreshes++ + else if (type === 'refreshError') state.refreshErrors++ + else if (type === 'protocolError') state.protocolErrors++ + else if (type === 'socketError') state.socketErrors++ + else if (type === 'spliceOpened') { + state.activeSplices++ + state.peakActiveSplices = Math.max(state.peakActiveSplices, state.activeSplices) + } else if (type === 'spliceCompleted') { + state.completedSplices++ + if (detail.readerMode === 'slow') state.slowReaderSplicesCompleted++ + } else if (type === 'spliceWedged') { + state.wedgedReaderSplicesClosed++ + const code = String(detail.code) + state.readerClosesByCode[code] = (state.readerClosesByCode[code] ?? 0) + 1 + } else if (type === 'spliceClosed') state.activeSplices-- + else if (type === 'spliceFailed') state.failedSplices++ + else if (type === 'shutdown') state.peerShutdowns++ +} + +async function connect(peer) { + try { + await peer.connect() + } catch (error) { + state.connectionFailures++ + if (state.steadyStarted) state.steadyConnectionFailures++ + else if (state.transitionWindow) state.transitionConnectionFailures++ + else state.rampConnectionFailures++ + const reason = relayLoadFailureReason(error) + state.connectionFailuresByReason[reason] = + (state.connectionFailuresByReason[reason] ?? 0) + 1 + scheduleReconnect(peer) + } +} + +const peerOptions = (index, overrides = {}) => ({ + ...config, + ...key, + accessToken, + ...(accessTokenProviderForIndex + ? { accessTokenProvider: accessTokenProviderForIndex(index) } + : {}), + seed: 0x4f524341 ^ config.shardIndex, + ...overrides +}) +if (config.regionBehaviorProbes > 0) { + const proofIndex = config.controls * config.shardCount + 10_000 + const regionProof = await proveRelayLoadRegionBehavior({ + oldClientPeer: new RelayLoadControlPeer( + proofIndex, + peerOptions(proofIndex, { preferredRegion: undefined }), + () => undefined + ), + stickyPeer: new RelayLoadControlPeer( + proofIndex + 1, + peerOptions(proofIndex + 1, { preferredRegion: 'asia-east2' }), + () => undefined + ), + asiaOrigin: config.capacityCellOrigin + }) + state.oldClientUsFirstProved = regionProof.oldClientUsFirst ? 1 : 0 + state.stickyAssignmentProved = regionProof.stickyAssignmentPreserved ? 1 : 0 +} +const initialConnections = [] +for (let localIndex = 0; localIndex < config.controls; localIndex++) { + const globalIndex = localIndex * config.shardCount + config.shardIndex + const peer = new RelayLoadControlPeer(globalIndex, peerOptions(globalIndex), observe) + peers.set(globalIndex, peer) + const rampOffset = + config.controls === 1 ? 0 : Math.floor((localIndex / (config.controls - 1)) * config.rampMs) + const offset = config.rampStartDelayMs + rampOffset + initialConnections.push(delay(offset).then(() => connect(peer))) +} +const progressTimer = setInterval(() => report(state), 10_000) +progressTimer.unref() +await runRelayLoadWithShutdown(async () => { + await Promise.all(initialConnections) + assertRelayLoadRampAccepted(state.rampConnectionFailures, config.maxRampConnectionFailures) + if ( + config.rebindProbes > 0 || config.placementOverflowProbes > 0 || + config.regionalFallbackProbes > 0 + ) { + state.transitionWindow = config.rebindDelayMs > 0 + await waitForRelayLoadRebindGate({ + delay, + delayMs: config.rebindDelayMs, + activeCount: () => state.active.size, + requiredCount: config.controls + }) + state.transitionWindow = false + } + if (config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0) { + const closesBeforeBoundary = state.closes + await waitForRelayLoadDirectorCapacity({ + directorOrigin: config.directorOrigin, + adminToken, + cellId: config.capacityCellId, + hardCap: config.capacityHardCap, + unobservedBound: config.capacityUnobservedBound, + requiredConnections: config.aggregateControls + }) + if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) { + throw new Error('ordinary controls changed during the director capacity gate') + } + const overflowIndex = config.controls * config.shardCount + config.shardIndex + if (config.placementOverflowProbes > 0) { + state.placementOverflowReason = await proveRelayLoadPlacementBoundary({ + peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), observe), + failureReason: relayLoadFailureReason + }) + } + if (config.regionalFallbackProbes > 0) { + await proveRelayLoadRegionalFallback({ + peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), () => undefined), + blockedOrigin: config.capacityCellOrigin + }) + state.regionalFallbacksProved = 1 + } + if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) { + throw new Error('ordinary controls changed during the placement boundary probe') + } + await waitForRelayLoadDirectorCapacity({ + directorOrigin: config.directorOrigin, + adminToken, + cellId: config.capacityCellId, + hardCap: config.capacityHardCap, + unobservedBound: config.capacityUnobservedBound, + requiredConnections: config.aggregateControls, + requiredSamples: 1 + }) + if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) { + throw new Error('ordinary controls changed before post-probe capacity verification') + } + } + const rebindResult = await proveRelayLoadRebindBoundary({ + peers: [...state.active].map((index) => peers.get(index)), + probeCount: config.rebindProbes, + holdMs: config.rebindHoldMs, + delay, + failureReason: relayLoadFailureReason, + requireOverflow: config.requireRebindOverflow + }) + state.rebindProbesOpened = rebindResult.opened + state.rebindOverflowReason = rebindResult.overflowReason + if (config.requestUnitInvites > 0) { + state.requestUnitInvitesOpened = await openRelayLoadInviteOffers({ + peers: [...state.active].sort((left, right) => left - right).map((index) => peers.get(index)), + count: config.requestUnitInvites, + ratePerSecond: config.requestUnitInvitesPerSecond + }) + } + if (config.requestUnitOverflowProbes > 0) { + await waitForRelayLoadRequestUnits({ + directorOrigin: config.directorOrigin, + adminToken, + cellId: config.capacityCellId, + capacityRequests: config.requestUnitCapacity, + expectedRequestUnits: config.requestUnitCapacity, + expectedActivityLeases: config.requestUnitCapacity + }) + state.requestUnitOverflowReason = await proveRelayLoadRequestUnitBoundary( + peers.get([...state.active][0]) + ) + } + if (config.phaseBarrierDir) { + await waitForRelayLoadPhaseBarrier({ + directory: config.phaseBarrierDir, + shardCount: config.shardCount, + shardIndex: config.shardIndex, + timeoutMs: config.phaseBarrierTimeoutMs + }) + state.phaseBarrierPassed = true + } + state.steadyStarted = true + state.steadyMinimumActive = state.active.size + const spliceIndexes = relayLoadSpliceIndexes(config) + const readerOrigins = spliceIndexes.flatMap((index, spliceIndex) => + relayLoadSpliceProfile(config, spliceIndex).readerMode === 'normal' + ? [] + : [peers.get(index).lastAssignment.cellUrl] + ) + state.readerEvidence = await createRelayLoadReaderEvidence(readerOrigins, { + readQueuedBytes: readRuntimeQueuedBytes, + delay + }) + if (config.phaseBarrierDir) { + await waitForRelayLoadPhaseBarrier({ + directory: `${config.phaseBarrierDir}-splices`, + shardCount: config.shardCount, + shardIndex: config.shardIndex, + timeoutMs: config.phaseBarrierTimeoutMs + }) + } + const splicePromises = spliceIndexes.map((index, spliceIndex) => + delay(relayLoadSpliceStartDelayMs(config, spliceIndex)).then(() => + peers.get(index).openSplice({ + payloadBytes: config.splicePayloadBytes, + ...relayLoadSpliceProfile(config, spliceIndex), + observeReaderPressure, + holdMs: config.spliceHoldMs + }) + ) + ) + await Promise.all([...splicePromises, delay(config.durationMs)]) +}, async () => { + state.stopping = true + clearInterval(progressTimer) + for (const timeout of reconnectTimers) clearTimeout(timeout) + reconnectTimers.clear() + await Promise.all([...peers.values()].map((peer) => peer.shutdown())) + eventLoopDelay.disable() + state.shutdownEvidence = { + peerShutdowns: state.peerShutdowns, + activeControls: state.active.size, + activeSplices: state.activeSplices, + reconnectTimers: reconnectTimers.size + } +}) +if (config.requestUnitCleanupTimeoutMs > 0) { + await waitForRelayLoadRequestUnits({ + directorOrigin: config.directorOrigin, + adminToken, + cellId: config.capacityCellId, + capacityRequests: config.requestUnitCapacity, + expectedRequestUnits: 0, + expectedActivityLeases: 0, + timeoutMs: config.requestUnitCleanupTimeoutMs + }) + state.requestUnitCleanupProved = 1 +} +const result = report(state, true) +const minimumPeak = config.allowPartial ? 1 : Math.ceil(config.controls * 0.95) +if (result.peakActive < minimumPeak) { + throw new Error(`peak active controls ${result.peakActive} below required ${minimumPeak}`) +} +if (result.steadyMinimumActive < minimumPeak) { + throw new Error( + `steady minimum active controls ${result.steadyMinimumActive} below required ${minimumPeak}` + ) +} +if (relayLoadRunHasDisallowedFailures(result, config)) { + throw new Error('relay load run observed connection, protocol, refresh, or socket errors') +} +const readerEvidenceError = relayLoadReaderEvidenceError(result, config) +if (readerEvidenceError) throw new Error(readerEvidenceError) +if ( + result.failedSplices > 0 || + result.completedSplices + result.wedgedReaderSplicesClosed !== config.splices || + result.shutdownEvidence.peerShutdowns !== config.controls || + result.shutdownEvidence.activeControls !== 0 || + result.shutdownEvidence.activeSplices !== 0 || + result.shutdownEvidence.reconnectTimers !== 0 +) { + throw new Error('relay load run did not complete splices or shut down cleanly') +} diff --git a/cloud/dev/scripts/operate-relay-asia-admission.mjs b/cloud/dev/scripts/operate-relay-asia-admission.mjs new file mode 100644 index 00000000000..45322bbe620 --- /dev/null +++ b/cloud/dev/scripts/operate-relay-asia-admission.mjs @@ -0,0 +1,440 @@ +import { createHash } from 'node:crypto' +import { fileURLToPath } from 'node:url' +import { + addExactMigrationCells, + applyExactAdmissionSelector, + inspectAdmissionSelector, + membershipWithStates, + selectorCellState +} from './relay-admission-selector.mjs' + +const SHAPES = { + staging: { + directorOrigin: 'https://relay-staging.onorca.dev', + domain: 'relay-staging.onorca.dev', + allCells: ['staging-gce-c4'] + }, + production: { + directorOrigin: 'https://relay.onorca.dev', + domain: 'relay.onorca.dev', + allCells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'] + } +} + +function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['environment', 'mode', 'cell-ids', 'image-digest']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (!/^sha256:[a-f0-9]{64}$/.test(values['image-digest'])) { + throw new Error('--image-digest is invalid') + } + if (![ + 'inspect', 'initialize', 'verify', 'registered', 'register', + 'promote', 'recover-promotion', 'rollback' + ].includes(values.mode)) { + throw new Error('--mode is invalid') + } + const expectedGeneration = values.mode === 'inspect' + ? undefined + : Number(values['expected-generation']) + if ( + values.mode !== 'inspect' && + (!Number.isSafeInteger(expectedGeneration) || expectedGeneration < 0) + ) { + throw new Error('--expected-generation is invalid') + } + const shape = SHAPES[values.environment] + if (!shape) throw new Error('--environment is invalid') + const cells = values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean) + const distinct = new Set(cells) + if (distinct.size !== cells.length || cells.some((cell) => !shape.allCells.includes(cell))) { + throw new Error('--cell-ids are invalid') + } + const exact = (expected) => JSON.stringify([...cells].sort()) === JSON.stringify([...expected].sort()) + if ( + (['inspect', 'initialize', 'register', 'registered', 'verify'].includes(values.mode) && + !exact(shape.allCells)) || + (['promote', 'recover-promotion'].includes(values.mode) && values.environment === 'production' && + !exact(['production-gce-c27']) && !exact(['production-gce-c28', 'production-gce-c29'])) || + (['promote', 'recover-promotion'].includes(values.mode) && values.environment === 'staging' && !exact(shape.allCells)) || + (values.mode === 'rollback' && cells.length === 0) + ) throw new Error('--cell-ids do not match the reviewed admission wave') + const attemptId = values['attempt-id'] + if (!['inspect', 'verify', 'registered'].includes(values.mode) && + !/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) { + throw new Error('--attempt-id is invalid') + } + return { + environment: values.environment, + mode: values.mode, + cells, + expectedGeneration, + expectedMembershipSha256: values['expected-membership-sha256'], + imageDigest: values['image-digest'], + attemptId, + token: process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '' + } +} + +function hostname(cellId) { + return cellId.split('-').at(-1) +} + +function cellOrigin(shape, cellId) { + return `https://${hostname(cellId)}.${shape.domain}` +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({})) + if (!response.ok) throw new Error(`${label} returned ${response.status}`) + return body +} + +function defaultPost(fetchImpl, token) { + return async (url, body) => await responseJson(await fetchImpl(url, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }), new URL(url).pathname) +} + +async function verifyRuntime(fetchImpl, post, shape, cellId, imageDigest, requireDirector) { + const origin = cellOrigin(shape, cellId) + const [health, ready, runtime] = await Promise.all([ + fetchImpl(`${origin}/health`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }), + fetchImpl(`${origin}/ready`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }), + post(`${origin}/v1/admin/runtime-status`, { v: 1 }) + ]) + if (!health.ok || !ready.ok) throw new Error(`${cellId} is not ready`) + if ( + runtime.cellId !== cellId || + runtime.cellUrl !== origin || + runtime.region !== 'asia-east2' || + runtime.imageDigest !== imageDigest || + runtime.draining !== false || + runtime.connectionCapacity?.hardCap !== 3_000 || + runtime.connectionCapacity?.unobservedBound !== 60 + ) throw new Error(`${cellId} runtime does not match the reviewed Asia shape`) + if (requireDirector) { + const result = await post(`${shape.directorOrigin}/v1/admin/cell-status`, { v: 1, cellId }) + if ( + result.status?.cellUrl !== origin || + result.status?.runtime?.heartbeatFresh !== true || + result.status?.runtime?.ready !== true + ) throw new Error(`${cellId} has no fresh ready director heartbeat`) + } +} + +function membershipStates(selector, cells) { + return Object.fromEntries(cells.map((cellId) => [cellId, selectorCellState(selector, cellId)])) +} + +function inspectedMembershipStates(selector, cells) { + const known = new Set([ + ...selector.membership.existingOnly, + ...selector.membership.migrationOnly, + ...selector.membership.general + ]) + return Object.fromEntries(cells.map((cellId) => [ + cellId, + known.has(cellId) ? selectorCellState(selector, cellId) : 'absent' + ])) +} + +function sameMembership(left, right) { + return JSON.stringify(left) === JSON.stringify(right) +} + +function membershipSha256(membership) { + return createHash('sha256').update(JSON.stringify(membership)).digest('hex') +} + +async function initializeAdmissionBoundary(post, selectorPost, shape, config, current) { + if (config.expectedGeneration !== 0) { + throw new Error('admission boundary initialization requires generation 0') + } + if ( + !/^[a-f0-9]{64}$/.test(config.expectedMembershipSha256 ?? '') || + membershipSha256(current.selector.membership) !== config.expectedMembershipSha256 + ) { + throw new Error('admission membership changed before boundary initialization') + } + const targetStates = inspectedMembershipStates(current.selector, config.cells) + if (Object.values(targetStates).some((state) => state !== 'absent')) { + throw new Error('Asia cell exists before admission boundary initialization') + } + const intendedMembership = current.intent?.previousMembership ?? current.selector.membership + const exactCommitted = (inspection) => + inspection.intent?.state === 'committed' && + inspection.intent.expectedGeneration === 0 && + inspection.selector.generation === 1 && + inspection.selector.attemptId === config.attemptId && + sameMembership(inspection.intent.previousMembership, intendedMembership) && + sameMembership(inspection.intent.membership, intendedMembership) && + sameMembership(inspection.selector.membership, intendedMembership) + const exactUnchanged = (inspection) => + inspection.intent?.state === 'unchanged' && + inspection.intent.expectedGeneration === 0 && + inspection.selector.generation === 0 && + sameMembership(inspection.intent.previousMembership, intendedMembership) && + sameMembership(inspection.intent.membership, intendedMembership) && + sameMembership(inspection.selector.membership, intendedMembership) + if (exactCommitted(current)) { + return { + mode: config.mode, + generation: current.selector.generation, + states: inspectedMembershipStates(current.selector, config.cells), + recovered: true + } + } + if (current.intent && !exactUnchanged(current)) { + throw new Error('admission boundary initialization attempt diverged') + } + const request = { + v: 1, + attemptId: config.attemptId, + expectedGeneration: 0, + expectedMembershipSha256: config.expectedMembershipSha256, + membership: intendedMembership + } + let applyError + for (let attempt = 0; attempt < 2; attempt++) { + try { + await post(`${shape.directorOrigin}/v1/admin/admission-selector/apply`, request) + } catch (error) { + applyError = error + } + const verified = await inspectAdmissionSelector(selectorPost, config.attemptId) + if (exactCommitted(verified)) { + return { + mode: config.mode, + generation: verified.selector.generation, + states: targetStates, + recovered: current.intent !== null || applyError !== undefined || attempt > 0 + } + } + if (!exactUnchanged(verified)) { + throw new Error('admission boundary initialization did not commit exactly', { + cause: applyError + }) + } + } + throw new Error('admission boundary initialization remained unchanged after retry', { + cause: applyError + }) +} + +export async function operateRelayAsiaAdmission(config, dependencies = {}) { + const shape = SHAPES[config.environment] + const fetchImpl = dependencies.fetch ?? fetch + const post = dependencies.post ?? defaultPost(fetchImpl, config.token) + const selectorPost = (path, body) => { + if (config.environment !== 'staging' || path !== '/v1/admin/admission-selector/apply') { + return post(`${shape.directorOrigin}${path}`, body) + } + const state = selectorCellState({ membership: body.membership }, 'staging-gce-c4') + if (!['general', 'migration-only'].includes(state)) { + throw new Error('staging proof can only transition C4 between reviewed states') + } + return post(`${shape.directorOrigin}/v1/admin/admission-selector/apply-staging-asia-proof`, { + v: 1, + attemptId: body.attemptId, + expectedGeneration: body.expectedGeneration, + state + }) + } + const current = await inspectAdmissionSelector( + selectorPost, + ['inspect', 'verify', 'registered'].includes(config.mode) ? undefined : config.attemptId + ) + if (config.mode === 'inspect') { + return { + mode: config.mode, + generation: current.selector.generation, + membership: current.selector.membership, + membershipSha256: membershipSha256(current.selector.membership), + states: inspectedMembershipStates(current.selector, config.cells) + } + } + if ( + !current.intent && + current.selector.generation !== config.expectedGeneration + ) { + throw new Error('admission selector generation changed') + } + if (current.intent && current.intent.expectedGeneration !== config.expectedGeneration) { + throw new Error('admission attempt generation does not match') + } + if (config.mode === 'initialize') { + return await initializeAdmissionBoundary(post, selectorPost, shape, config, current) + } + if (config.mode === 'recover-promotion') { + if (config.cells.every( + (cellId) => selectorCellState(current.selector, cellId) === 'migration-only' + )) { + return { + mode: config.mode, + promoted: false, + generation: current.selector.generation, + states: membershipStates(current.selector, config.cells) + } + } + if (!current.intent) { + if ( + current.selector.generation !== config.expectedGeneration + ) throw new Error('promotion state changed without the reviewed attempt') + return { + mode: config.mode, + promoted: false, + generation: current.selector.generation, + states: membershipStates(current.selector, config.cells) + } + } + const expectedMembership = membershipWithStates( + { membership: current.intent.previousMembership }, + Object.fromEntries(config.cells.map((cellId) => [cellId, 'general'])) + ) + if ( + current.intent.state !== 'committed' || + JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) || + config.cells.some((cellId) => selectorCellState(current.selector, cellId) !== 'general') + ) throw new Error('promotion attempt is not the current general state') + return { + mode: config.mode, + promoted: true, + generation: current.selector.generation, + states: membershipStates(current.selector, config.cells) + } + } + if (config.mode === 'rollback') { + for (const cellId of config.cells) { + if (!['general', 'migration-only'].includes(selectorCellState(current.selector, cellId))) { + throw new Error(`${cellId} cannot roll back to migration-only`) + } + } + } else if (config.mode === 'register') { + const known = new Set([ + ...current.selector.membership.existingOnly, + ...current.selector.membership.migrationOnly, + ...current.selector.membership.general + ]) + if (!current.intent && config.cells.some((cellId) => known.has(cellId))) { + throw new Error('Asia cell is already registered') + } + await Promise.all(config.cells.map((cellId) => + verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false) + )) + } else if (config.mode !== 'registered') { + await Promise.all(config.cells.map((cellId) => + verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, true) + )) + } + if (config.mode === 'registered') { + if (config.cells.some( + (cellId) => selectorCellState(current.selector, cellId) !== 'migration-only' + )) throw new Error('Asia cells are not registered migration-only') + await Promise.all(config.cells.map((cellId) => + verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false) + )) + } + if (['verify', 'registered'].includes(config.mode)) { + return { + mode: config.mode, + generation: current.selector.generation, + states: membershipStates(current.selector, config.cells) + } + } + if (config.mode === 'register') { + if (current.intent) { + const expectedCells = new Set(config.cells) + const addedCells = current.intent.membership.migrationOnly.filter( + (cellId) => !current.intent.previousMembership.migrationOnly.includes(cellId) + ) + if ( + current.intent.state !== 'committed' || + addedCells.length !== expectedCells.size || + addedCells.some((cellId) => !expectedCells.has(cellId)) || + current.selector.generation !== config.expectedGeneration + 1 || + JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership) + ) { + throw new Error('admission attempt does not match the requested Asia registration') + } + return { + mode: config.mode, + generation: current.selector.generation, + states: membershipStates(current.selector, config.cells), + recovered: true + } + } + const result = await addExactMigrationCells( + selectorPost, + { + attemptId: config.attemptId, + cells: config.cells.map((cellId) => ({ + cellId, + cellUrl: cellOrigin(shape, cellId), + region: 'asia-east2', + capacityRequests: 6_000, + connectionHardCap: 3_000, + connectionUnobservedBound: 60 + })) + }, + { expectedCurrentSelector: current.selector } + ) + return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) } + } + const desiredState = config.mode === 'promote' ? 'general' : 'migration-only' + if (current.intent) { + const expectedMembership = membershipWithStates( + { membership: current.intent.previousMembership }, + Object.fromEntries(config.cells.map((cellId) => [cellId, desiredState])) + ) + if ( + current.intent.state !== 'committed' || + JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) || + current.selector.generation !== config.expectedGeneration + 1 || + JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership) + ) { + throw new Error('admission attempt does not match the requested Asia transition') + } + return { + mode: config.mode, + generation: current.selector.generation, + states: membershipStates(current.selector, config.cells), + recovered: true + } + } + if (config.mode === 'promote' && config.cells.some( + (cellId) => selectorCellState(current.selector, cellId) !== 'migration-only' + )) throw new Error('Asia promotion requires migration-only cells') + if ( + config.mode === 'promote' && + config.environment === 'production' && + config.cells.includes('production-gce-c28') && + selectorCellState(current.selector, 'production-gce-c27') !== 'general' + ) { + throw new Error('Asia expansion requires the C27 canary to be general') + } + const result = await applyExactAdmissionSelector( + selectorPost, + membershipWithStates(current.selector, Object.fromEntries( + config.cells.map((cellId) => [cellId, desiredState]) + )), + { attemptId: config.attemptId, expectedCurrentSelector: current.selector } + ) + return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const config = parseArguments(process.argv.slice(2)) + if (!config.token) throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required') + console.log(JSON.stringify(await operateRelayAsiaAdmission(config))) +} diff --git a/cloud/dev/scripts/operate-relay-asia-admission.test.mjs b/cloud/dev/scripts/operate-relay-asia-admission.test.mjs new file mode 100644 index 00000000000..7f23bb1f8e6 --- /dev/null +++ b/cloud/dev/scripts/operate-relay-asia-admission.test.mjs @@ -0,0 +1,512 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { test } from 'node:test' +import { operateRelayAsiaAdmission } from './operate-relay-asia-admission.mjs' + +const digest = `sha256:${'a'.repeat(64)}` +const membershipDigest = (membership) => + createHash('sha256').update(JSON.stringify(membership)).digest('hex') + +function harness(initialSelector) { + const initialMembership = structuredClone(initialSelector.membership) + let selector = structuredClone(initialSelector) + const intents = new Map() + const requests = [] + let fetches = 0 + let failAfterIntent = false + const post = async (url, body) => { + const parsed = new URL(url) + requests.push({ path: parsed.pathname, body }) + if (parsed.pathname === '/v1/admin/runtime-status') { + const cell = parsed.hostname.split('.')[0] + return { + cellId: `production-gce-${cell}`, + cellUrl: parsed.origin, + region: 'asia-east2', + imageDigest: digest, + draining: false, + connectionCapacity: { hardCap: 3_000, unobservedBound: 60 } + } + } + if (parsed.pathname === '/v1/admin/cell-status') { + return { + status: { + cellUrl: `https://${body.cellId.split('-').at(-1)}.relay.onorca.dev`, + runtime: { heartbeatFresh: true, ready: true } + } + } + } + if (parsed.pathname.endsWith('/status')) { + return { selector, intent: body.attemptId ? intents.get(body.attemptId) ?? null : null } + } + if (parsed.pathname.endsWith('/add-migration-cells')) { + selector = { + generation: selector.generation + 1, + attemptId: body.attemptId, + membership: { + ...selector.membership, + migrationOnly: [...selector.membership.migrationOnly, ...body.cells.map((cell) => cell.cellId)].sort() + } + } + } else if (parsed.pathname.endsWith('/apply-staging-asia-proof')) { + const membership = structuredClone(selector.membership) + membership.migrationOnly = membership.migrationOnly.filter((cell) => cell !== 'staging-gce-c4') + membership.general = membership.general.filter((cell) => cell !== 'staging-gce-c4') + membership[body.state === 'general' ? 'general' : 'migrationOnly'].push('staging-gce-c4') + selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership } + } else if (parsed.pathname.endsWith('/apply')) { + if ( + body.expectedMembershipSha256 && + body.expectedMembershipSha256 !== membershipDigest(selector.membership) + ) throw new Error('admission_selector_membership_mismatch') + if (failAfterIntent) { + failAfterIntent = false + intents.set(body.attemptId, { + state: 'unchanged', + expectedGeneration: body.expectedGeneration, + previousMembership: structuredClone(selector.membership), + membership: structuredClone(body.membership) + }) + throw new Error('failure after intent persistence') + } + selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership: body.membership } + } else throw new Error(`unexpected ${parsed.pathname}`) + intents.set(body.attemptId, { + state: 'committed', expectedGeneration: body.expectedGeneration, + previousMembership: initialSelector.membership, + membership: selector.membership + }) + return { changed: true, selector } + } + const fetch = async () => { + fetches++ + return new Response(null, { status: 200 }) + } + const commitWithoutResponse = async (path, body) => { + await post(`https://relay.onorca.dev${path}`, body) + throw new Error('response lost after commit') + } + return { + post, fetch, requests, commitWithoutResponse, + failNextApplyAfterIntent: () => (failAfterIntent = true), + apply: async (attemptId, membership) => await post( + 'https://relay.onorca.dev/v1/admin/admission-selector/apply', + { attemptId, expectedGeneration: selector.generation, membership } + ), + fetchCount: () => fetches, selector: () => selector + } +} + +const baseSelector = { + generation: 7, + membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26'] } +} + +test('inspects generation zero without requiring target registration or making a mutation', async () => { + const subject = harness({ + generation: 0, + membership: { + existingOnly: ['staging-gce-c3'], + migrationOnly: [], + general: ['staging-gce-c1', 'staging-gce-c2'] + } + }) + const result = await operateRelayAsiaAdmission({ + environment: 'staging', mode: 'inspect', cells: ['staging-gce-c4'], + imageDigest: digest, token: 'not-logged' + }, subject) + assert.equal(result.generation, 0) + assert.equal(result.states['staging-gce-c4'], 'absent') + assert.deepEqual(result.membership, subject.selector().membership) + assert.equal(result.membershipSha256, membershipDigest(subject.selector().membership)) + assert.deepEqual(subject.requests.map(({ path }) => path), [ + '/v1/admin/admission-selector/status' + ]) +}) + +test('initializes generation zero without changing membership', async () => { + const membership = { + existingOnly: ['staging-gce-c3'], + migrationOnly: [], + general: ['staging-gce-c1', 'staging-gce-c2'] + } + const subject = harness({ generation: 0, membership }) + const result = await operateRelayAsiaAdmission({ + environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], + expectedGeneration: 0, imageDigest: digest, + expectedMembershipSha256: membershipDigest(membership), + attemptId: 'asia_boundary_0', token: 'not-logged' + }, subject) + const request = subject.requests.find(({ path }) => path.endsWith('/apply')) + assert.deepEqual(request.body, { + v: 1, + attemptId: 'asia_boundary_0', + expectedGeneration: 0, + expectedMembershipSha256: membershipDigest(membership), + membership + }) + assert.equal(result.generation, 1) + assert.equal(result.states['staging-gce-c4'], 'absent') + assert.deepEqual(subject.selector().membership, membership) +}) + +test('retries the same fingerprint-bound initialization after intent persistence', async () => { + const membership = { + existingOnly: ['staging-gce-c3'], + migrationOnly: [], + general: ['staging-gce-c1', 'staging-gce-c2'] + } + const subject = harness({ generation: 0, membership }) + subject.failNextApplyAfterIntent() + const result = await operateRelayAsiaAdmission({ + environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], + expectedGeneration: 0, imageDigest: digest, + expectedMembershipSha256: membershipDigest(membership), + attemptId: 'asia_boundary_intent_retry', token: 'not-logged' + }, subject) + const applies = subject.requests.filter(({ path }) => path.endsWith('/apply')) + assert.equal(applies.length, 2) + assert.deepEqual(applies[1].body, applies[0].body) + assert.equal(result.recovered, true) + assert.equal(result.generation, 1) + assert.deepEqual(subject.selector().membership, membership) +}) + +test('recovers a committed generation-zero initialization', async () => { + const membership = { + existingOnly: ['staging-gce-c3'], + migrationOnly: [], + general: ['staging-gce-c1', 'staging-gce-c2'] + } + const subject = harness({ generation: 0, membership }) + const config = { + environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], + expectedGeneration: 0, imageDigest: digest, + expectedMembershipSha256: membershipDigest(membership), + attemptId: 'asia_boundary_retry', token: 'not-logged' + } + await assert.rejects(subject.commitWithoutResponse( + '/v1/admin/admission-selector/apply', + { + v: 1, + attemptId: config.attemptId, + expectedGeneration: 0, + expectedMembershipSha256: membershipDigest(membership), + membership + } + ), /response lost after commit/) + const recovered = await operateRelayAsiaAdmission(config, subject) + assert.equal(recovered.recovered, true) + assert.equal(recovered.generation, 1) + assert.deepEqual(subject.selector().membership, membership) +}) + +test('rejects generation-zero membership drift after inspect', async () => { + const inspected = { + existingOnly: ['staging-gce-c3'], + migrationOnly: [], + general: ['staging-gce-c1', 'staging-gce-c2'] + } + const changed = { + existingOnly: ['staging-gce-c2', 'staging-gce-c3'], + migrationOnly: [], + general: ['staging-gce-c1'] + } + const subject = harness({ generation: 0, membership: changed }) + await assert.rejects(operateRelayAsiaAdmission({ + environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], + expectedGeneration: 0, imageDigest: digest, + expectedMembershipSha256: membershipDigest(inspected), + attemptId: 'asia_boundary_drift', token: 'not-logged' + }, subject), /membership changed/) + assert.equal(subject.requests.some(({ path }) => path.endsWith('/apply')), false) +}) + +test('registers all three Asia cells atomically with region and exact limits', async () => { + const subject = harness(baseSelector) + const result = await operateRelayAsiaAdmission({ + environment: 'production', mode: 'register', + cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], + expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_7', token: 'not-logged' + }, subject) + const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) + assert.equal(request.body.cells.length, 3) + assert.ok(request.body.cells.every((cell) => + cell.region === 'asia-east2' && cell.capacityRequests === 6_000 && + cell.connectionHardCap === 3_000 && cell.connectionUnobservedBound === 60 + )) + assert.equal(result.generation, 8) + assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only'])) +}) + +test('promotes the canary only after runtime and director-heartbeat checks', async () => { + const subject = harness({ + generation: 8, + membership: { existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] } + }) + const result = await operateRelayAsiaAdmission({ + environment: 'production', mode: 'promote', cells: ['production-gce-c27'], + expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_8', token: 'not-logged' + }, subject) + assert.equal(subject.fetchCount(), 2) + assert.equal(result.states['production-gce-c27'], 'general') +}) + +test('checks registered migration-only cells before director configuration without requiring heartbeat', async () => { + const subject = harness({ + generation: 8, + membership: { + existingOnly: [], + migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], + general: ['production-gce-c26'] + } + }) + const result = await operateRelayAsiaAdmission({ + environment: 'production', mode: 'registered', + cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], + expectedGeneration: 8, imageDigest: digest, token: 'not-logged' + }, subject) + assert.equal(subject.fetchCount(), 6) + assert.equal(subject.requests.filter(({ path }) => path === '/v1/admin/cell-status').length, 0) + assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only'])) +}) + +test('rolls back admission without requiring an unhealthy runtime to answer', async () => { + const subject = harness({ + generation: 9, + membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] } + }) + const result = await operateRelayAsiaAdmission({ + environment: 'production', mode: 'rollback', cells: ['production-gce-c27'], + expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_9', token: 'not-logged' + }, subject) + assert.equal(subject.fetchCount(), 0) + assert.equal(result.states['production-gce-c27'], 'migration-only') +}) + +test('uses the server-enforced C4-only route for staging proof transitions', async () => { + const subject = harness({ + generation: 3, + membership: { + existingOnly: ['staging-gce-c1'], + migrationOnly: [], + general: ['staging-gce-c2', 'staging-gce-c4'] + } + }) + const result = await operateRelayAsiaAdmission({ + environment: 'staging', mode: 'rollback', cells: ['staging-gce-c4'], + expectedGeneration: 3, imageDigest: digest, attemptId: 'asia_staging_rollback', + token: 'not-logged' + }, subject) + const request = subject.requests.find( + ({ path }) => path.endsWith('/apply-staging-asia-proof') + ) + assert.deepEqual(request.body, { + v: 1, + attemptId: 'asia_staging_rollback', + expectedGeneration: 3, + state: 'migration-only' + }) + assert.deepEqual(subject.selector().membership, { + existingOnly: ['staging-gce-c1'], + migrationOnly: ['staging-gce-c4'], + general: ['staging-gce-c2'] + }) + assert.equal(result.states['staging-gce-c4'], 'migration-only') +}) + +test('fails closed when the exact selector generation moved', async () => { + const subject = harness(baseSelector) + await assert.rejects(operateRelayAsiaAdmission({ + environment: 'production', mode: 'register', + cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], + expectedGeneration: 6, imageDigest: digest, attemptId: 'asia_register_6', token: 'not-logged' + }, subject), /generation changed/) + assert.equal(subject.fetchCount(), 0) +}) + +test('recovers a committed registration when the workflow retries the original generation', async () => { + const subject = harness(baseSelector) + const config = { + environment: 'production', mode: 'register', + cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], + expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_retry', + token: 'not-logged' + } + await assert.rejects(subject.commitWithoutResponse( + '/v1/admin/admission-selector/add-migration-cells', + { + v: 1, + attemptId: config.attemptId, + expectedGeneration: config.expectedGeneration, + cells: config.cells.map((cellId) => ({ cellId })) + } + ), /response lost after commit/) + const recovered = await operateRelayAsiaAdmission(config, subject) + assert.equal(recovered.recovered, true) + assert.equal(recovered.generation, 8) +}) + +test('recovers a committed promotion when the workflow retries the original generation', async () => { + const subject = harness({ + generation: 8, + membership: { + existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] + } + }) + const config = { + environment: 'production', mode: 'promote', cells: ['production-gce-c27'], + expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_retry', + token: 'not-logged' + } + await assert.rejects(subject.commitWithoutResponse( + '/v1/admin/admission-selector/apply', + { + v: 1, + attemptId: config.attemptId, + expectedGeneration: config.expectedGeneration, + membership: { + existingOnly: [], migrationOnly: [], + general: ['production-gce-c26', 'production-gce-c27'] + } + } + ), /response lost after commit/) + const recovered = await operateRelayAsiaAdmission(config, subject) + assert.equal(recovered.recovered, true) + assert.equal(recovered.generation, 9) + assert.equal(recovered.states['production-gce-c27'], 'general') +}) + +test('inspects an ambiguous promotion without creating a new transition', async () => { + const untouched = harness({ + generation: 8, + membership: { + existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] + } + }) + const config = { + environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'], + expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_promote', + token: 'not-logged' + } + const absent = await operateRelayAsiaAdmission(config, untouched) + assert.equal(absent.promoted, false) + assert.equal(untouched.requests.some(({ path }) => path.endsWith('/apply')), false) + + const committed = harness({ + generation: 8, + membership: { + existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] + } + }) + await assert.rejects(committed.commitWithoutResponse('/v1/admin/admission-selector/apply', { + v: 1, + attemptId: config.attemptId, + expectedGeneration: 8, + membership: { + existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] + } + }), /response lost after commit/) + const recovered = await operateRelayAsiaAdmission(config, committed) + assert.equal(recovered.promoted, true) + assert.equal(recovered.generation, 9) +}) + +test('treats an already rolled-back promotion as recovered', async () => { + const subject = harness({ + generation: 8, + membership: { + existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] + } + }) + const config = { + environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'], + expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_after_rollback', + token: 'not-logged' + } + await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', { + v: 1, attemptId: config.attemptId, expectedGeneration: 8, + membership: { + existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] + } + }), /response lost after commit/) + await subject.apply('later_rollback', { + existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] + }) + const recovered = await operateRelayAsiaAdmission(config, subject) + assert.equal(recovered.promoted, false) + assert.equal(recovered.generation, 10) +}) + +test('recovers a committed rollback when the workflow retries the original generation', async () => { + const subject = harness({ + generation: 9, + membership: { + existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] + } + }) + const config = { + environment: 'production', mode: 'rollback', cells: ['production-gce-c27'], + expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_retry', + token: 'not-logged' + } + await assert.rejects(subject.commitWithoutResponse( + '/v1/admin/admission-selector/apply', + { + v: 1, + attemptId: config.attemptId, + expectedGeneration: config.expectedGeneration, + membership: { + existingOnly: [], migrationOnly: ['production-gce-c27'], + general: ['production-gce-c26'] + } + } + ), /response lost after commit/) + const recovered = await operateRelayAsiaAdmission(config, subject) + assert.equal(recovered.recovered, true) + assert.equal(recovered.generation, 10) + assert.equal(recovered.states['production-gce-c27'], 'migration-only') +}) + +test('rejects a committed transition retry after a later selector change', async () => { + const subject = harness({ + generation: 8, + membership: { + existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] + } + }) + const config = { + environment: 'production', mode: 'promote', cells: ['production-gce-c27'], + expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_stale_promote', + token: 'not-logged' + } + await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', { + v: 1, attemptId: config.attemptId, expectedGeneration: 8, + membership: { + existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] + } + }), /response lost after commit/) + await subject.apply('later_rollback', { + existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] + }) + await assert.rejects( + operateRelayAsiaAdmission(config, subject), + /does not match the requested Asia transition/ + ) +}) + +test('requires the C27 canary before promoting C28 and C29', async () => { + const subject = harness({ + generation: 8, + membership: { + existingOnly: [], + migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], + general: ['production-gce-c26'] + } + }) + await assert.rejects(operateRelayAsiaAdmission({ + environment: 'production', mode: 'promote', + cells: ['production-gce-c28', 'production-gce-c29'], expectedGeneration: 8, + imageDigest: digest, attemptId: 'asia_wave_before_canary', token: 'not-logged' + }, subject), /C27 canary/) +}) diff --git a/cloud/dev/scripts/operate-relay-regional-rehome.mjs b/cloud/dev/scripts/operate-relay-regional-rehome.mjs new file mode 100644 index 00000000000..2ccce39926d --- /dev/null +++ b/cloud/dev/scripts/operate-relay-regional-rehome.mjs @@ -0,0 +1,312 @@ +import { pathToFileURL } from 'node:url' +import { inspectAdmissionSelector } from './relay-admission-selector.mjs' + +const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' +const MODES = new Set(['inspect', 'enable', 'pause', 'disable', 'recover-enable']) + +function canonicalCells(value) { + if (value === 'none') return [] + const cells = value.split(',').map((cell) => cell.trim()).filter(Boolean).sort() + if ( + cells.length === 0 || + new Set(cells).size !== cells.length || + cells.some((cell) => !/^production-gce-c(?:[1-9]|[12][0-9])$/.test(cell)) + ) throw new Error('selector membership is invalid') + return cells +} + +function integer(value, name, { minimum = 0, maximum = Number.MAX_SAFE_INTEGER } = {}) { + const parsed = Number(value) + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new Error(`${name} is invalid`) + } + return parsed +} + +export function parseRegionalRehomeArguments(argv, environment = process.env) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['mode', 'director-origin', 'expected-control-generation']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (!MODES.has(values.mode)) throw new Error('--mode is invalid') + if (values['director-origin'] !== DIRECTOR_ORIGIN) { + throw new Error('--director-origin must be the production Relay origin') + } + const recovery = values.mode === 'recover-enable' + const mutation = values.mode !== 'inspect' && !recovery + const mutationKeys = [ + 'not-before', + 'rate-per-minute', + 'preference-max-age-ms', + 'drain-grace-ms', + 'confirmation' + ] + if (mutation && mutationKeys.some((key) => values[key] === undefined)) { + throw new Error('mutations require the complete durable control shape') + } + if (!mutation && !recovery && mutationKeys.some((key) => values[key] !== undefined)) { + throw new Error('inspect cannot carry mutation arguments') + } + const selectorKeys = [ + 'expected-selector-generation', + 'expected-existing-only-cells', + 'expected-migration-only-cells', + 'expected-general-cells' + ] + if (!recovery && selectorKeys.some((key) => values[key] === undefined)) { + throw new Error('operation requires exact selector state') + } + if (recovery && selectorKeys.some((key) => values[key] !== undefined)) { + throw new Error('enable recovery cannot depend on selector diagnostics') + } + const expectedConfirmation = { + enable: 'ENABLE_REGIONAL_REHOMING', + pause: 'PAUSE_REGIONAL_REHOMING', + disable: 'DISABLE_REGIONAL_REHOMING' + }[values.mode] + if (mutation && values.confirmation !== expectedConfirmation) { + throw new Error('confirmation does not match the requested control action') + } + if (recovery && values.confirmation !== 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE') { + throw new Error('confirmation does not authorize failed-enable recovery') + } + if ( + recovery && + mutationKeys + .filter((key) => key !== 'confirmation') + .some((key) => values[key] !== undefined) + ) throw new Error('enable recovery cannot carry durable control shape arguments') + const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN + if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') + return { + mode: values.mode, + directorOrigin: DIRECTOR_ORIGIN, + ...(!recovery + ? { + expectedSelectorGeneration: integer( + values['expected-selector-generation'], + '--expected-selector-generation' + ), + expectedMembership: { + existingOnly: canonicalCells(values['expected-existing-only-cells']), + migrationOnly: canonicalCells(values['expected-migration-only-cells']), + general: canonicalCells(values['expected-general-cells']) + } + } + : {}), + expectedControlGeneration: integer( + values['expected-control-generation'], + '--expected-control-generation' + ), + ...(mutation + ? { + notBefore: integer(values['not-before'], '--not-before'), + ratePerMinute: integer(values['rate-per-minute'], '--rate-per-minute', { + minimum: 1, + maximum: 120 + }), + preferenceMaxAgeMs: integer( + values['preference-max-age-ms'], + '--preference-max-age-ms', + { minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 } + ), + drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', { + minimum: 60_000, + maximum: 60 * 60_000 + }) + } + : {}), + token + } +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({})) + if (!response.ok) throw new Error(`${label} returned ${response.status}: ${body.error ?? 'unknown'}`) + return body +} + +function exactMembership(actual, expected) { + return ['existingOnly', 'migrationOnly', 'general'].every( + (key) => JSON.stringify(actual[key]) === JSON.stringify(expected[key]) + ) +} + +function assertControl(control, expected) { + if ( + (expected.generation !== undefined && control?.generation !== expected.generation) || + typeof control.enabled !== 'boolean' || + !Number.isSafeInteger(control.observationStartedAt) || + !Number.isSafeInteger(control.notBefore) || + !Number.isSafeInteger(control.ratePerMinute) || + !Number.isSafeInteger(control.preferenceMaxAgeMs) || + !Number.isSafeInteger(control.drainGraceMs) + ) throw new Error('director returned an invalid regional rehome control') + if (expected.enabled !== undefined && control.enabled !== expected.enabled) { + throw new Error('regional rehome enabled state does not match') + } + return control +} + +async function verifiedDisabledControl(post, generation) { + return assertControl((await post('/v1/admin/regional-rehome-control', { + v: 1, + action: 'inspect' + })).control, { generation, enabled: false }) +} + +async function applyDisabledControl(post, before) { + return assertControl((await post('/v1/admin/regional-rehome-control', { + v: 1, + action: 'apply', + expectedGeneration: before.generation, + enabled: false, + notBefore: before.notBefore, + ratePerMinute: before.ratePerMinute, + preferenceMaxAgeMs: before.preferenceMaxAgeMs, + drainGraceMs: before.drainGraceMs, + confirmation: 'DISABLE_REGIONAL_REHOMING' + })).control, { generation: before.generation + 1, enabled: false }) +} + +async function resolveAmbiguousDisable(post, before, firstError) { + const observed = assertControl((await post('/v1/admin/regional-rehome-control', { + v: 1, + action: 'inspect' + })).control, {}) + if (observed.generation === before.generation + 1 && !observed.enabled) { + return observed + } + if (observed.generation !== before.generation || !observed.enabled) { + throw new AggregateError( + [firstError], + 'failed-enable recovery reached an unexpected control generation' + ) + } + try { + return await applyDisabledControl(post, before) + } catch (retryError) { + try { + return await verifiedDisabledControl(post, before.generation + 1) + } catch (readbackError) { + throw new AggregateError( + [firstError, retryError, readbackError], + 'failed-enable recovery exhausted two bounded CAS attempts' + ) + } + } +} + +export async function recoverRegionalRehomeEnable(config, post) { + const before = assertControl((await post('/v1/admin/regional-rehome-control', { + v: 1, + action: 'inspect' + })).control, {}) + if ( + before.generation < config.expectedControlGeneration || + (before.generation === config.expectedControlGeneration && before.enabled) + ) throw new Error('durable control cannot belong to the failed enable attempt') + if (!before.enabled) { + const verified = await verifiedDisabledControl(post, before.generation) + return { mode: config.mode, recovered: false, control: verified } + } + let applied + try { + applied = await applyDisabledControl(post, before) + } catch (error) { + applied = await resolveAmbiguousDisable(post, before, error) + } + const verified = await verifiedDisabledControl(post, applied.generation) + return { mode: config.mode, recovered: true, control: verified } +} + +export async function operateRegionalRehome(config, dependencies = {}) { + const fetchImpl = dependencies.fetch ?? fetch + const post = dependencies.post ?? (async (path, body) => await responseJson( + await fetchImpl(`${config.directorOrigin}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${config.token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }), + path + )) + if (config.mode === 'recover-enable') { + return await recoverRegionalRehomeEnable(config, post) + } + const selector = (await inspectAdmissionSelector(post)).selector + if ( + selector.generation !== config.expectedSelectorGeneration || + !exactMembership(selector.membership, config.expectedMembership) + ) throw new Error('admission selector does not match the reviewed generation and membership') + + const inspected = await post('/v1/admin/regional-rehome-control', { + v: 1, + action: 'inspect' + }) + const before = assertControl(inspected.control, { + generation: config.expectedControlGeneration + }) + if (config.mode === 'inspect') return { mode: config.mode, selector, control: before } + if (config.mode === 'enable' && before.enabled) { + throw new Error('regional rehome is already enabled; inspect before changing its rate') + } + if (config.mode === 'pause' && !before.enabled) { + throw new Error('regional rehome is already paused') + } + const enabled = config.mode === 'enable' + const applied = await post('/v1/admin/regional-rehome-control', { + v: 1, + action: 'apply', + expectedGeneration: config.expectedControlGeneration, + enabled, + notBefore: config.notBefore, + ratePerMinute: config.ratePerMinute, + preferenceMaxAgeMs: config.preferenceMaxAgeMs, + drainGraceMs: config.drainGraceMs, + confirmation: enabled + ? 'ENABLE_REGIONAL_REHOMING' + : 'DISABLE_REGIONAL_REHOMING' + }) + const after = assertControl(applied.control, { + generation: config.expectedControlGeneration + 1, + enabled + }) + const verified = assertControl((await post('/v1/admin/regional-rehome-control', { + v: 1, + action: 'inspect' + })).control, { + generation: after.generation, + enabled + }) + return { mode: config.mode, selector, control: verified } +} + +export async function main( + argv = process.argv.slice(2), + environment = process.env, + dependencies = {}, + write = (value) => process.stdout.write(value) +) { + const result = await operateRegionalRehome( + parseRegionalRehomeArguments(argv, environment), + dependencies + ) + write(`${JSON.stringify({ event: 'relay_regional_rehome_control', ...result })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs b/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs new file mode 100644 index 00000000000..8ffe38dfe09 --- /dev/null +++ b/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs @@ -0,0 +1,265 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + main, + operateRegionalRehome, + parseRegionalRehomeArguments, + recoverRegionalRehomeEnable +} from './operate-relay-regional-rehome.mjs' + +const membership = { + existingOnly: ['production-gce-c1'], + migrationOnly: ['production-gce-c2'], + general: ['production-gce-c7', 'production-gce-c27'] +} + +function argumentsFor(mode, confirmation) { + return [ + '--mode', mode, + '--director-origin', 'https://relay.onorca.dev', + '--expected-selector-generation', '11', + '--expected-existing-only-cells', membership.existingOnly.join(','), + '--expected-migration-only-cells', membership.migrationOnly.join(','), + '--expected-general-cells', membership.general.join(','), + '--expected-control-generation', '4', + ...(mode === 'inspect' ? [] : [ + '--not-before', '2000000000000', + '--rate-per-minute', '10', + '--preference-max-age-ms', '86400000', + '--drain-grace-ms', '60000', + '--confirmation', confirmation + ]) + ] +} + +function control(generation, enabled) { + return { + generation, + enabled, + observationStartedAt: 1, + notBefore: 2_000_000_000_000, + ratePerMinute: 10, + preferenceMaxAgeMs: 86_400_000, + drainGraceMs: 60_000 + } +} + +test('parses exact selector and typed control confirmation', () => { + const parsed = parseRegionalRehomeArguments( + argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ) + assert.equal(parsed.expectedSelectorGeneration, 11) + assert.equal(parsed.expectedControlGeneration, 4) + assert.equal(parsed.ratePerMinute, 10) + assert.throws( + () => parseRegionalRehomeArguments( + argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ), + /confirmation/ + ) + assert.throws( + () => parseRegionalRehomeArguments( + argumentsFor('inspect').concat('--rate-per-minute', '10'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ), + /inspect cannot/ + ) +}) + +test('binds enable to exact selector and durable control generations', async () => { + const requests = [] + const controls = [ + { generation: 4, enabled: false }, + { generation: 5, enabled: true }, + { generation: 5, enabled: true } + ].map((control) => ({ + observationStartedAt: 1, + notBefore: 0, + ratePerMinute: 10, + preferenceMaxAgeMs: 86_400_000, + drainGraceMs: 60_000, + ...control + })) + const config = parseRegionalRehomeArguments( + argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ) + const result = await operateRegionalRehome(config, { + post: async (path, body) => { + requests.push({ path, body }) + if (path === '/v1/admin/admission-selector/status') { + return { selector: { generation: 11, membership } } + } + return { v: 1, control: controls.shift() } + } + }) + assert.equal(result.control.generation, 5) + assert.deepEqual(requests[2].body, { + v: 1, + action: 'apply', + expectedGeneration: 4, + enabled: true, + notBefore: 2_000_000_000_000, + ratePerMinute: 10, + preferenceMaxAgeMs: 86_400_000, + drainGraceMs: 60_000, + confirmation: 'ENABLE_REGIONAL_REHOMING' + }) +}) + +test('fails closed on selector drift before reading or mutating control', async () => { + let calls = 0 + const config = parseRegionalRehomeArguments( + argumentsFor('disable', 'DISABLE_REGIONAL_REHOMING'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ) + await assert.rejects( + operateRegionalRehome(config, { + post: async () => { + calls += 1 + return { selector: { generation: 12, membership } } + } + }), + /selector/ + ) + assert.equal(calls, 1) +}) + +test('failed-enable recovery CAS-disables an advanced enabled generation', async () => { + const requests = [] + let current = control(7, true) + const result = await recoverRegionalRehomeEnable({ + mode: 'recover-enable', + expectedControlGeneration: 4 + }, async (_path, body) => { + requests.push(body) + if (body.action === 'inspect') return { control: current } + assert.equal(body.expectedGeneration, 7) + current = control(8, false) + throw new Error('enable recovery response was lost') + }) + assert.equal(result.recovered, true) + assert.deepEqual(result.control, control(8, false)) + assert.deepEqual(requests[1], { + v: 1, + action: 'apply', + expectedGeneration: 7, + enabled: false, + notBefore: 2_000_000_000_000, + ratePerMinute: 10, + preferenceMaxAgeMs: 86_400_000, + drainGraceMs: 60_000, + confirmation: 'DISABLE_REGIONAL_REHOMING' + }) +}) + +test('failed-enable recovery retries once when the first CAS never commits', async () => { + let current = control(7, true) + const applyRequests = [] + const result = await recoverRegionalRehomeEnable({ + mode: 'recover-enable', + expectedControlGeneration: 4 + }, async (_path, body) => { + if (body.action === 'inspect') return { control: current } + applyRequests.push(body) + if (applyRequests.length === 1) { + throw new Error('disable request was lost before commit') + } + current = control(8, false) + throw new Error('retry response was lost after commit') + }) + assert.equal(applyRequests.length, 2) + assert.deepEqual(applyRequests[1], applyRequests[0]) + assert.equal(result.recovered, true) + assert.deepEqual(result.control, control(8, false)) +}) + +test('failed-enable recovery stops after two uncommitted CAS attempts', async () => { + let applyCalls = 0 + await assert.rejects( + recoverRegionalRehomeEnable({ + mode: 'recover-enable', + expectedControlGeneration: 4 + }, async (_path, body) => { + if (body.action === 'inspect') return { control: control(7, true) } + applyCalls += 1 + throw new Error(`disable attempt ${applyCalls} was lost before commit`) + }), + /exhausted two bounded CAS attempts/ + ) + assert.equal(applyCalls, 2) +}) + +test('failed-enable recovery is a verified no-op before enable and after cleanup', async () => { + for (const current of [control(4, false), control(8, false)]) { + const requests = [] + const result = await recoverRegionalRehomeEnable({ + mode: 'recover-enable', + expectedControlGeneration: 4 + }, async (_path, body) => { + requests.push(body) + return { control: current } + }) + assert.equal(result.recovered, false) + assert.equal(result.control.enabled, false) + assert.deepEqual(requests.map(({ action }) => action), ['inspect', 'inspect']) + } +}) + +test('failed-enable recovery rejects an unchanged pre-existing enabled state', async () => { + await assert.rejects( + recoverRegionalRehomeEnable({ + mode: 'recover-enable', + expectedControlGeneration: 4 + }, async () => ({ control: control(4, true) })), + /cannot belong to the failed enable attempt/ + ) +}) + +test('parses recovery without depending on selector diagnostics', () => { + const recoveryArguments = [ + '--mode', 'recover-enable', + '--director-origin', 'https://relay.onorca.dev', + '--expected-control-generation', '4', + '--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE' + ] + const parsed = parseRegionalRehomeArguments( + recoveryArguments, + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ) + assert.equal(parsed.expectedControlGeneration, 4) + assert.equal(parsed.expectedMembership, undefined) + assert.throws( + () => parseRegionalRehomeArguments( + recoveryArguments.concat('--not-before', '2000000000000'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ), + /cannot carry durable control shape/ + ) +}) + +test('main executes recovery mode and emits verified disabled control', async () => { + let current = control(5, true) + let output = '' + await main([ + '--mode', 'recover-enable', + '--director-origin', 'https://relay.onorca.dev', + '--expected-control-generation', '4', + '--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE' + ], { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }, { + post: async (_path, body) => { + if (body.action === 'apply') current = control(6, false) + return { control: current } + } + }, (value) => { + output += value + }) + assert.deepEqual(JSON.parse(output), { + event: 'relay_regional_rehome_control', + mode: 'recover-enable', + recovered: true, + control: control(6, false) + }) +}) diff --git a/cloud/dev/scripts/power-staging-relay.mjs b/cloud/dev/scripts/power-staging-relay.mjs new file mode 100644 index 00000000000..55704294192 --- /dev/null +++ b/cloud/dev/scripts/power-staging-relay.mjs @@ -0,0 +1,487 @@ +import { readFileSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { pathToFileURL } from 'node:url' +import { + inspectAdmissionSelector, + selectorCellState +} from './relay-admission-selector.mjs' + +const PROJECT = 'onorca-cloud-staging' +const REGION = 'us-central1' +const DIRECTOR_ORIGIN = 'https://relay-staging.onorca.dev' +const ADMIN_AUDIENCE = `${DIRECTOR_ORIGIN}/v1/admin/drain` +const SQL_INSTANCE = 'orca-cloud-staging-auth-db' +const CLOUD_RUN_SERVICES = [ + { name: 'orca-cloud-relay-staging', healthOrigin: DIRECTOR_ORIGIN }, + { name: 'orca-cloud-auth-staging', healthOrigin: 'https://auth-staging.onorca.dev' } +] +const POLL_INTERVAL_MS = 5_000 +const WAKE_TIMEOUT_MS = 12 * 60 * 1_000 + +export function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) + const name = key.slice(2) + if (!['mode', 'wake-cells', 'topology-file'].includes(name)) { + throw new Error(`unsupported argument --${name}`) + } + values[name] = value + } + if (!['status', 'sleep', 'wake'].includes(values.mode)) { + throw new Error('--mode must be status, sleep, or wake') + } + if (!['configured', 'all'].includes(values['wake-cells'])) { + throw new Error('--wake-cells must be configured or all') + } + if (!values['topology-file']) throw new Error('missing --topology-file') + return { + mode: values.mode, + wakeCells: values['wake-cells'], + topologyFile: values['topology-file'] + } +} + +function canonicalStagingCell(cellId, value) { + if (!/^staging-gce-[a-z0-9-]+$/.test(cellId)) throw new Error(`unsafe staging cell ID ${cellId}`) + if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`) + const cell = { + cellId, + migName: String(value.mig_name ?? ''), + zone: String(value.zone ?? ''), + origin: String(value.origin ?? ''), + initiallyEnabled: value.initially_enabled + } + if (!/^orca-cloud-staging-relay-gce-[a-z0-9-]+$/.test(cell.migName)) { + throw new Error(`${cellId} has an unsafe MIG name`) + } + if (!/^(?:us-central1|asia-east2)-[a-z]$/.test(cell.zone)) { + throw new Error(`${cellId} has an unsafe zone`) + } + const origin = new URL(cell.origin) + if ( + origin.protocol !== 'https:' || + origin.origin !== cell.origin || + !origin.hostname.endsWith('.relay-staging.onorca.dev') + ) { + throw new Error(`${cellId} has an unsafe origin`) + } + if (typeof cell.initiallyEnabled !== 'boolean') { + throw new Error(`${cellId} has no initial admission state`) + } + return cell +} + +export function readStagingTopology(file) { + const parsed = JSON.parse(readFileSync(file, 'utf8')) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('staging topology must be an object') + } + const cells = Object.entries(parsed) + .map(([cellId, value]) => canonicalStagingCell(cellId, value)) + .sort((left, right) => left.cellId.localeCompare(right.cellId)) + if (cells.length < 2 || cells.length > 10) { + throw new Error('staging topology must contain 2..10 cells') + } + if (cells.filter((cell) => cell.initiallyEnabled).length < 2) { + throw new Error('staging topology must retain two configured admission cells') + } + return cells +} + +function defaultCommand(args, json) { + const result = spawnSync('gcloud', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'] + }) + if (result.status !== 0) { + throw new Error(`gcloud ${args.slice(0, 5).join(' ')} failed: ${result.stderr.trim()}`) + } + return json ? JSON.parse(result.stdout) : result.stdout.trim() +} + +function suppliedAdminToken(environment = process.env) { + const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN + if (!token || token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) { + throw new Error('workflow did not supply a valid masked staging admin token') + } + return token +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({ error: `http_${response.status}` })) + if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`) + return body +} + +function createAdminPost(deps) { + const token = deps.adminToken() + return async (origin, path, body) => + await responseJson( + await deps.fetch(`${origin}${path}`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }), + path + ) +} + +async function waitUntil(deps, label, operation, timeoutMs = WAKE_TIMEOUT_MS) { + const deadline = deps.now() + timeoutMs + let lastError + while (deps.now() < deadline) { + try { + const result = await operation() + if (result) return result + } catch (error) { + lastError = error + } + await deps.wait(POLL_INTERVAL_MS) + } + const detail = lastError instanceof Error ? `: ${lastError.message}` : '' + throw new Error(`timed out waiting for ${label}${detail}`) +} + +async function checkHealth(deps, origin, path) { + const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) }) + const body = await response.json().catch(() => ({})) + return response.ok && body.ok === true +} + +function sqlActivationPolicy(instance) { + return String(instance.settings?.activationPolicy ?? '') +} + +function describeSql(deps) { + return deps.commandJson([ + 'sql', + 'instances', + 'describe', + SQL_INSTANCE, + '--project', + PROJECT, + '--format=json' + ]) +} + +async function ensureSqlPolicy(deps, policy) { + if (sqlActivationPolicy(describeSql(deps)) === policy) return false + deps.command([ + 'sql', + 'instances', + 'patch', + SQL_INSTANCE, + '--project', + PROJECT, + `--activation-policy=${policy}`, + '--quiet' + ]) + await waitUntil(deps, `Cloud SQL activation policy ${policy}`, () => + sqlActivationPolicy(describeSql(deps)) === policy + ) + return true +} + +function describeMig(deps, cell) { + return deps.commandJson([ + 'compute', + 'instance-groups', + 'managed', + 'describe', + cell.migName, + '--project', + PROJECT, + '--zone', + cell.zone, + '--format=json' + ]) +} + +async function setMigSize(deps, cell, size) { + const before = describeMig(deps, cell) + if (Number(before.targetSize) !== size) { + deps.command([ + 'compute', + 'instance-groups', + 'managed', + 'resize', + cell.migName, + '--project', + PROJECT, + '--zone', + cell.zone, + `--size=${size}`, + '--quiet' + ]) + } + await waitUntil(deps, `${cell.cellId} size ${size}`, () => { + const current = describeMig(deps, cell) + return Number(current.targetSize) === size && current.status?.isStable === true + }) +} + +function activeRevisionName(service) { + const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0) + if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) { + throw new Error('Cloud Run service must have exactly one active revision') + } + return active[0].revisionName +} + +function describeRunService(deps, name) { + return deps.commandJson([ + 'run', + 'services', + 'describe', + name, + '--project', + PROJECT, + '--region', + REGION, + '--format=json' + ]) +} + +function describeRunRevision(deps, name) { + return deps.commandJson([ + 'run', + 'revisions', + 'describe', + name, + '--project', + PROJECT, + '--region', + REGION, + '--format=json' + ]) +} + +function revisionMinimum(revision) { + return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0) +} + +async function ensureCloudRunScaleToZero(deps, service) { + const before = describeRunService(deps, service.name) + const activeRevision = describeRunRevision(deps, activeRevisionName(before)) + if (revisionMinimum(activeRevision) === 0) return false + + const latestName = before.status?.latestReadyRevisionName + const latest = latestName ? describeRunRevision(deps, latestName) : null + if (!latest || revisionMinimum(latest) !== 0) { + deps.command([ + 'run', + 'services', + 'update', + service.name, + '--project', + PROJECT, + '--region', + REGION, + '--min-instances=0', + '--quiet' + ]) + } + deps.command([ + 'run', + 'services', + 'update-traffic', + service.name, + '--project', + PROJECT, + '--region', + REGION, + '--to-latest', + '--quiet' + ]) + await waitUntil(deps, `${service.name} scale-to-zero revision`, async () => { + const current = describeRunService(deps, service.name) + const revision = describeRunRevision(deps, activeRevisionName(current)) + return revisionMinimum(revision) === 0 && (await checkHealth(deps, service.healthOrigin, '/health')) + }) + return true +} + +async function cellStatus(adminPost, cell) { + const response = await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-status', { + v: 1, + cellId: cell.cellId + }) + if (!response.status || response.status.cellId !== cell.cellId) { + throw new Error(`${cell.cellId} returned an invalid status`) + } + return response.status +} + +function assertQuiescent(status) { + const active = { + activityLeases: status.activityLeases, + activityRequestUnits: status.activityRequestUnits, + outgoingMigrations: status.outgoingMigrations, + incomingMigrations: status.incomingMigrations, + observedRequests: status.runtime?.observedRequests ?? 0 + } + if (Object.values(active).some((value) => Number(value) !== 0)) { + throw new Error(`${status.cellId} still has active Relay work: ${JSON.stringify(active)}`) + } +} + +async function setCellState(adminPost, cell, enabled) { + await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-state', { + v: 1, + cellId: cell.cellId, + enabled + }) +} + +async function stagingStatus(deps, cells) { + return { + event: 'staging_relay_power_status', + project: PROJECT, + sqlActivationPolicy: sqlActivationPolicy(describeSql(deps)), + cells: cells.map((cell) => ({ + cellId: cell.cellId, + initiallyEnabled: cell.initiallyEnabled, + targetSize: Number(describeMig(deps, cell).targetSize) + })), + cloudRun: CLOUD_RUN_SERVICES.map((service) => { + const described = describeRunService(deps, service.name) + const revision = describeRunRevision(deps, activeRevisionName(described)) + return { service: service.name, activeRevisionMinimum: revisionMinimum(revision) } + }) + } +} + +async function sleepStaging(deps, cells) { + const sqlPolicy = sqlActivationPolicy(describeSql(deps)) + if (sqlPolicy === 'NEVER') { + const runningCells = cells.filter((cell) => Number(describeMig(deps, cell).targetSize) !== 0) + if (runningCells.length > 0) { + // SQL-off plus running workers is an unknown partial state; never kill those workers blindly. + throw new Error( + `staging is partially asleep with running cells: ${runningCells.map((cell) => cell.cellId).join(', ')}` + ) + } + deps.emit({ event: 'staging_relay_sleep_reconciled', alreadyAsleep: true }) + return + } + + const adminPost = createAdminPost(deps) + const initial = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell))) + for (const status of initial) assertQuiescent(status) + const selector = await inspectAdmissionSelector( + async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body) + ) + if (selector.selector.generation > 0) { + throw new Error( + 'staging sleep cannot reverse the monotonic admission selector; keep staging awake' + ) + } + const previouslyEnabled = new Set(initial.filter((status) => status.enabled).map((status) => status.cellId)) + + for (const cell of cells) await setCellState(adminPost, cell, false) + await deps.wait(15_000) + try { + const disabled = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell))) + for (const status of disabled) { + if (status.enabled) throw new Error(`${status.cellId} admission did not disable`) + assertQuiescent(status) + } + for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service) + const final = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell))) + for (const status of final) assertQuiescent(status) + } catch (error) { + for (const cell of cells.filter((candidate) => previouslyEnabled.has(candidate.cellId))) { + await setCellState(adminPost, cell, true).catch(() => undefined) + } + throw error + } + + await Promise.all(cells.map((cell) => setMigSize(deps, cell, 0))) + await ensureSqlPolicy(deps, 'NEVER') + deps.emit({ event: 'staging_relay_slept', stoppedCells: cells.map((cell) => cell.cellId) }) +} + +async function wakeStaging(deps, cells, wakeCells) { + await ensureSqlPolicy(deps, 'ALWAYS') + await waitUntil(deps, 'staging director health', () => checkHealth(deps, DIRECTOR_ORIGIN, '/health')) + for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service) + + const adminPost = createAdminPost(deps) + const selector = await inspectAdmissionSelector( + async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body) + ) + const selectorActive = selector.selector.generation > 0 + // Existing-only cells may still own live or dormant assignments, so a + // selector-era wake restores the complete retained topology. + const selected = selectorActive + ? cells + : cells.filter((cell) => wakeCells === 'all' || cell.initiallyEnabled) + await Promise.all( + cells.map((cell) => setMigSize(deps, cell, selected.includes(cell) ? 1 : 0)) + ) + for (const cell of selected) { + await waitUntil(deps, `${cell.cellId} health`, () => checkHealth(deps, cell.origin, '/health')) + await waitUntil(deps, `${cell.cellId} readiness`, () => checkHealth(deps, cell.origin, '/ready')) + } + + for (const cell of cells) { + if (selectorActive) { + const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => { + const current = await cellStatus(adminPost, cell) + return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null + }) + if (status.admissionState !== selectorCellState(selector.selector, cell.cellId)) { + throw new Error(`${cell.cellId} admission does not match selector`) + } + continue + } + if (!selected.includes(cell)) { + await setCellState(adminPost, cell, false) + continue + } + const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => { + const current = await cellStatus(adminPost, cell) + return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null + }) + if (cell.initiallyEnabled && !status.enabled) await setCellState(adminPost, cell, true) + if (!cell.initiallyEnabled && status.enabled) await setCellState(adminPost, cell, false) + } + deps.emit({ + event: 'staging_relay_woke', + runningCells: selected.map((cell) => cell.cellId), + admissionCells: selectorActive + ? selector.selector.membership.general + : selected.filter((cell) => cell.initiallyEnabled).map((cell) => cell.cellId) + }) +} + +export async function runStagingRelayPower(config, overrides = {}) { + const deps = { + command: overrides.command ?? ((args) => defaultCommand(args, false)), + commandJson: overrides.commandJson ?? ((args) => defaultCommand(args, true)), + fetch: overrides.fetch ?? fetch, + adminToken: overrides.adminToken ?? suppliedAdminToken, + emit: overrides.emit ?? ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)), + now: overrides.now ?? Date.now, + wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) + } + const cells = readStagingTopology(config.topologyFile) + if (config.mode === 'status') deps.emit(await stagingStatus(deps, cells)) + else if (config.mode === 'sleep') await sleepStaging(deps, cells) + else await wakeStaging(deps, cells, config.wakeCells) +} + +export async function main(argv = process.argv.slice(2)) { + await runStagingRelayPower(parseArguments(argv)) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/power-staging-relay.test.mjs b/cloud/dev/scripts/power-staging-relay.test.mjs new file mode 100644 index 00000000000..21c7794d48e --- /dev/null +++ b/cloud/dev/scripts/power-staging-relay.test.mjs @@ -0,0 +1,359 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { + parseArguments, + readStagingTopology, + runStagingRelayPower +} from './power-staging-relay.mjs' + +function topologyFile(overrides = {}) { + const directory = mkdtempSync(join(tmpdir(), 'staging-relay-power-')) + const topology = { + 'staging-gce-c1': { + mig_name: 'orca-cloud-staging-relay-gce-c1', + zone: 'us-central1-b', + origin: 'https://c1.relay-staging.onorca.dev', + initially_enabled: true + }, + 'staging-gce-c2': { + mig_name: 'orca-cloud-staging-relay-gce-c2', + zone: 'us-central1-c', + origin: 'https://c2.relay-staging.onorca.dev', + initially_enabled: true + }, + 'staging-gce-c3': { + mig_name: 'orca-cloud-staging-relay-gce-c3', + zone: 'us-central1-a', + origin: 'https://c3.relay-staging.onorca.dev', + initially_enabled: false + }, + 'staging-gce-c4': { + mig_name: 'orca-cloud-staging-relay-gce-c4', + zone: 'asia-east2-a', + origin: 'https://c4.relay-staging.onorca.dev', + initially_enabled: false + }, + ...overrides + } + const file = join(directory, 'topology.json') + writeFileSync(file, JSON.stringify(topology)) + return file +} + +function argumentConfig(file, mode, wakeCells = 'configured') { + return parseArguments([ + '--mode', + mode, + '--wake-cells', + wakeCells, + '--topology-file', + file + ]) +} + +function response(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +function harness({ + sqlPolicy = 'ALWAYS', + migSize = 1, + observedRequests = 0, + selectorGeneration = 0 +} = {}) { + const cells = new Map( + ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'].map((cellId, index) => [ + cellId, + { + enabled: index < 2, + targetSize: migSize, + observedRequests, + initiallyEnabled: index < 2 + } + ]) + ) + const revisions = new Map([ + ['orca-cloud-relay-staging', { active: 'relay-00001', latest: 'relay-00001', min: 1 }], + ['orca-cloud-auth-staging', { active: 'auth-00001', latest: 'auth-00001', min: 1 }] + ]) + const revisionMinimums = new Map([ + ['relay-00001', 1], + ['auth-00001', 1] + ]) + const commands = [] + const events = [] + let activationPolicy = sqlPolicy + let clock = 0 + + function cellForMig(name) { + return [...cells.entries()].find(([, value], index) => { + const suffix = `c${index + 1}` + return name.endsWith(suffix) && value + }) + } + + function commandJson(args) { + if (args[0] === 'sql') return { settings: { activationPolicy } } + if (args[0] === 'compute') { + const entry = cellForMig(args[4]) + return { targetSize: entry[1].targetSize, status: { isStable: true } } + } + if (args[0] === 'run' && args[1] === 'services') { + const state = revisions.get(args[3]) + return { + status: { + latestReadyRevisionName: state.latest, + traffic: [{ percent: 100, revisionName: state.active }] + } + } + } + if (args[0] === 'run' && args[1] === 'revisions') { + return { + metadata: { + annotations: { + 'autoscaling.knative.dev/minScale': String(revisionMinimums.get(args[3]) ?? 0) + } + } + } + } + throw new Error(`unexpected JSON command ${args.join(' ')}`) + } + + function command(args) { + commands.push(args) + if (args[0] === 'sql') { + activationPolicy = args.find((arg) => arg.startsWith('--activation-policy='))?.split('=')[1] + return + } + if (args[0] === 'compute') { + const entry = cellForMig(args[4]) + entry[1].targetSize = Number(args.find((arg) => arg.startsWith('--size='))?.split('=')[1]) + return + } + if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update') { + const state = revisions.get(args[3]) + state.latest = `${args[3]}-power` + revisionMinimums.set(state.latest, 0) + return + } + if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update-traffic') { + const state = revisions.get(args[3]) + state.active = state.latest + return + } + throw new Error(`unexpected command ${args.join(' ')}`) + } + + async function fetchImpl(url, options = {}) { + const parsed = new URL(url) + if (!options.method) return response({ ok: true }) + const body = JSON.parse(options.body) + if (parsed.pathname === '/v1/admin/cell-status') { + const state = cells.get(body.cellId) + const index = [...cells.keys()].indexOf(body.cellId) + return response({ + v: 1, + status: { + cellId: body.cellId, + enabled: state.enabled, + admissionState: + selectorGeneration > 0 + ? index === 0 + ? 'existing-only' + : index === 1 + ? 'migration-only' + : index === 2 + ? 'general' + : 'migration-only' + : state.enabled + ? 'general' + : 'existing-only', + assignments: 0, + reservedRequests: 0, + activityLeases: 0, + activityRequestUnits: 0, + outgoingMigrations: 0, + incomingMigrations: 0, + runtime: { + ready: state.targetSize === 1, + heartbeatFresh: state.targetSize === 1, + observedRequests: state.observedRequests + } + } + }) + } + if (parsed.pathname === '/v1/admin/admission-selector/status') { + return response({ + v: 1, + selector: { + generation: selectorGeneration, + attemptId: null, + membership: { + existingOnly: + selectorGeneration > 0 + ? ['staging-gce-c1'] + : [...cells] + .filter(([, state]) => !state.enabled) + .map(([cellId]) => cellId), + migrationOnly: + selectorGeneration > 0 ? ['staging-gce-c2', 'staging-gce-c4'] : [], + general: + selectorGeneration > 0 + ? ['staging-gce-c3'] + : [...cells] + .filter(([, state]) => state.enabled) + .map(([cellId]) => cellId) + } + }, + intent: null + }) + } + if (parsed.pathname === '/v1/admin/cell-state') { + cells.get(body.cellId).enabled = body.enabled + return response({ ok: true }) + } + throw new Error(`unexpected fetch ${parsed.pathname}`) + } + + return { + cells, + commands, + events, + deps: { + command, + commandJson, + fetch: fetchImpl, + adminToken: () => 'header.payload.signature', + emit: (event) => events.push(event), + now: () => clock, + wait: async (ms) => { + clock += ms + } + }, + sqlPolicy: () => activationPolicy + } +} + +test('accepts only explicit staging power arguments and topology', () => { + const file = topologyFile() + assert.equal(argumentConfig(file, 'status').mode, 'status') + assert.equal(readStagingTopology(file).at(-1).zone, 'asia-east2-a') + assert.throws(() => argumentConfig(file, 'destroy')) + assert.throws(() => parseArguments(['--mode', 'sleep', '--wake-cells', 'configured'])) + + const unsafe = topologyFile({ + 'staging-gce-c1': { + mig_name: 'orca-cloud-relay-gce-c1', + zone: 'us-central1-a', + origin: 'https://c1.relay.onorca.dev', + initially_enabled: true + } + }) + assert.throws(() => readStagingTopology(unsafe), /unsafe/) + + const unreviewedRegion = topologyFile({ + 'staging-gce-c4': { + mig_name: 'orca-cloud-staging-relay-gce-c4', + zone: 'europe-west1-b', + origin: 'https://c4.relay-staging.onorca.dev', + initially_enabled: false + } + }) + assert.throws(() => readStagingTopology(unreviewedRegion), /unsafe zone/) +}) + +test('refuses sleep before changing admission when a cell has active requests', async () => { + const testHarness = harness({ observedRequests: 1 }) + await assert.rejects( + runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps), + /still has active Relay work/ + ) + assert.equal(testHarness.commands.length, 0) + assert.equal(testHarness.cells.get('staging-gce-c1').enabled, true) +}) + +test('sleeps only after disabling admission and proving zero active work', async () => { + const testHarness = harness() + await runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps) + + assert.equal(testHarness.sqlPolicy(), 'NEVER') + assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [0, 0, 0, 0]) + assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [false, false, false, false]) + assert.equal(testHarness.events.at(-1).event, 'staging_relay_slept') + assert.equal( + testHarness.commands.filter((args) => args[0] === 'run' && args[2] === 'update').length, + 2 + ) +}) + +test('refuses staging sleep after the monotonic selector boundary', async () => { + const testHarness = harness({ selectorGeneration: 1 }) + await assert.rejects( + runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps), + /cannot reverse the monotonic admission selector/ + ) + assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1]) +}) + +test('refuses to terminate workers from an unknown partially asleep state', async () => { + const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 1 }) + await assert.rejects( + runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps), + /partially asleep with running cells/ + ) + assert.equal(testHarness.commands.length, 0) + assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1]) +}) + +test('wakes SQL and configured cells while leaving the candidate off and disabled', async () => { + const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 0 }) + for (const cell of testHarness.cells.values()) cell.enabled = false + for (const state of testHarness.cells.values()) state.observedRequests = 0 + await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps) + + assert.equal(testHarness.sqlPolicy(), 'ALWAYS') + assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 0, 0]) + assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [true, true, false, false]) + assert.deepEqual(testHarness.events.at(-1), { + event: 'staging_relay_woke', + runningCells: ['staging-gce-c1', 'staging-gce-c2'], + admissionCells: ['staging-gce-c1', 'staging-gce-c2'] + }) +}) + +test('wakes every retained cell without rewriting selector-era admission', async () => { + const testHarness = harness({ + sqlPolicy: 'NEVER', + migSize: 0, + selectorGeneration: 1 + }) + const states = [...testHarness.cells.values()] + states[0].enabled = false + states[1].enabled = true + states[2].enabled = true + states[3].enabled = true + await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps) + + assert.deepEqual(states.map((cell) => cell.targetSize), [1, 1, 1, 1]) + assert.deepEqual(states.map((cell) => cell.enabled), [false, true, true, true]) + assert.deepEqual(testHarness.events.at(-1), { + event: 'staging_relay_woke', + runningCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'], + admissionCells: ['staging-gce-c3'] + }) +}) + +test('status is read-only and reports the current billable floor controls', async () => { + const testHarness = harness() + await runStagingRelayPower(argumentConfig(topologyFile(), 'status'), testHarness.deps) + assert.equal(testHarness.commands.length, 0) + assert.equal(testHarness.events[0].project, 'onorca-cloud-staging') + assert.equal(testHarness.events[0].sqlActivationPolicy, 'ALWAYS') + assert.equal(testHarness.events[0].cells.length, 4) +}) diff --git a/cloud/dev/scripts/prepare-relay-asia-director-cells.mjs b/cloud/dev/scripts/prepare-relay-asia-director-cells.mjs new file mode 100644 index 00000000000..cd39a83c549 --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-asia-director-cells.mjs @@ -0,0 +1,78 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +function argumentsFrom(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['current-json', 'topology-json', 'output', 'cell-ids', 'image-digest']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + return values +} + +export function prepareRelayAsiaDirectorCells({ currentCells, topology, cellIds, imageDigest }) { + if (!Array.isArray(currentCells) || !topology || Array.isArray(topology)) { + throw new Error('director inputs are invalid') + } + const additions = cellIds.split(',').map((value) => value.trim()).filter(Boolean) + if ( + additions.length === 0 || + new Set(additions).size !== additions.length + ) throw new Error('director Asia additions are invalid') + const currentIds = new Set(currentCells.map((cell) => cell.id)) + if (currentIds.size !== currentCells.length) throw new Error('current director cells contain duplicates') + const normalizedCurrent = currentCells.map((cell) => ({ + ...cell, + region: cell.region ?? 'us-central1' + })) + const desiredCells = additions.map((cellId) => { + const cell = topology[cellId] + if ( + !cell || + cell.region !== 'asia-east2' || + cell.capacity_requests !== 6_000 || + cell.database_pool_max !== 10 || + cell.connection_hard_cap !== 3_000 || + cell.connection_unobserved_bound !== 60 || + cell.initially_enabled !== false || + cell.image?.split('@')[1] !== imageDigest + ) throw new Error(`${cellId} state output does not match the reviewed Asia shape`) + return { + id: cellId, + url: cell.origin, + capacityRequests: cell.capacity_requests, + region: cell.region, + initiallyEnabled: false, + connectionHardCap: cell.connection_hard_cap, + connectionUnobservedBound: cell.connection_unobserved_bound + } + }) + const desiredById = new Map(desiredCells.map((cell) => [cell.id, cell])) + for (const current of normalizedCurrent) { + const desired = desiredById.get(current.id) + if (!desired) continue + for (const [key, value] of Object.entries(desired)) { + if (current[key] !== value) { + throw new Error(`${current.id} director configuration differs from the reviewed Asia shape`) + } + } + } + const configured = new Set(normalizedCurrent.map((cell) => cell.id)) + return [...normalizedCurrent, ...desiredCells.filter((cell) => !configured.has(cell.id))] +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const values = argumentsFrom(process.argv.slice(2)) + const result = prepareRelayAsiaDirectorCells({ + currentCells: JSON.parse(readFileSync(values['current-json'], 'utf8')), + topology: JSON.parse(readFileSync(values['topology-json'], 'utf8')), + cellIds: values['cell-ids'], + imageDigest: values['image-digest'] + }) + writeFileSync(values.output, `${JSON.stringify(result)}\n`, { mode: 0o600 }) +} diff --git a/cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs b/cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs new file mode 100644 index 00000000000..9a223725006 --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { prepareRelayAsiaDirectorCells } from './prepare-relay-asia-director-cells.mjs' + +const digest = `sha256:${'a'.repeat(64)}` +const topologyCell = (ordinal, zone) => ({ + origin: `https://c${ordinal}.relay.onorca.dev`, region: 'asia-east2', zone, + capacity_requests: 6_000, database_pool_max: 10, + connection_hard_cap: 3_000, connection_unobserved_bound: 60, + initially_enabled: false, + image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@${digest}` +}) + +test('preserves current order, defaults predecessor regions, and appends exact Asia cells', () => { + const current = [{ + id: 'production-gce-c1', url: 'https://c1.relay.onorca.dev', + capacityRequests: 4_000, initiallyEnabled: false + }] + const result = prepareRelayAsiaDirectorCells({ + currentCells: current, + topology: { + 'production-gce-c27': topologyCell(27, 'asia-east2-a'), + 'production-gce-c28': topologyCell(28, 'asia-east2-b'), + 'production-gce-c29': topologyCell(29, 'asia-east2-c') + }, + cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', + imageDigest: digest + }) + assert.equal(result[0].region, 'us-central1') + assert.deepEqual(result.slice(1).map(({ id }) => id), [ + 'production-gce-c27', 'production-gce-c28', 'production-gce-c29' + ]) + assert.ok(result.slice(1).every((cell) => + cell.region === 'asia-east2' && cell.initiallyEnabled === false && + cell.connectionHardCap === 3_000 + )) +}) + +test('is idempotent for an exact existing Asia cell and rejects director drift', () => { + const topology = { 'production-gce-c27': topologyCell(27, 'asia-east2-a') } + const current = prepareRelayAsiaDirectorCells({ + currentCells: [], topology, cellIds: 'production-gce-c27', imageDigest: digest + }) + assert.deepEqual(prepareRelayAsiaDirectorCells({ + currentCells: current, topology, cellIds: 'production-gce-c27', imageDigest: digest + }), current) + assert.throws(() => prepareRelayAsiaDirectorCells({ + currentCells: [{ ...current[0], capacityRequests: 5_999 }], + topology, cellIds: 'production-gce-c27', imageDigest: digest + }), /director configuration differs/) +}) + +test('rejects a mismatching topology state output', () => { + const wrong = topologyCell(27, 'asia-east2-a') + wrong.database_pool_max = 20 + assert.throws(() => prepareRelayAsiaDirectorCells({ + currentCells: [], topology: { 'production-gce-c27': wrong }, + cellIds: 'production-gce-c27', imageDigest: digest + }), /does not match/) +}) diff --git a/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs b/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs new file mode 100644 index 00000000000..db76a83ede0 --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs @@ -0,0 +1,113 @@ +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +const BOOT_IMAGE = 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21' +const SHAPES = { + staging: { project: 'onorca-cloud-staging', cells: { 'staging-gce-c4': 'asia-east2-a' } }, + production: { + project: 'onorca-cloud', + cells: { + 'production-gce-c27': 'asia-east2-a', + 'production-gce-c28': 'asia-east2-b', + 'production-gce-c29': 'asia-east2-c' + } + } +} + +function argumentsFrom(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['existing-json', 'environment', 'cell-ids', 'image']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + return values +} + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical) + if (value && typeof value === 'object') { + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])) + } + return value +} + +export function prepareRelayAsiaTopologyInput({ + existingCells, + existingAdditionalRegions, + environment, + cellIds, + image +}) { + const shape = SHAPES[environment] + if (!shape) throw new Error('invalid environment') + const requested = cellIds.split(',').map((value) => value.trim()).filter(Boolean).sort() + const expected = Object.keys(shape.cells).sort() + if (new Set(requested).size !== requested.length || JSON.stringify(requested) !== JSON.stringify(expected)) { + throw new Error('cell IDs do not match the reviewed Asia topology') + } + const prefix = `us-central1-docker.pkg.dev/${shape.project}/orca-cloud/relay@sha256:` + if (!image.startsWith(prefix) || !/sha256:[a-f0-9]{64}$/.test(image)) { + throw new Error('image is not the environment Relay image pinned by digest') + } + if (!existingCells || Array.isArray(existingCells) || typeof existingCells !== 'object') { + throw new Error('existing Relay cells must be an object') + } + if ( + !existingAdditionalRegions || + Array.isArray(existingAdditionalRegions) || + typeof existingAdditionalRegions !== 'object' || + JSON.stringify(canonical(existingAdditionalRegions)) !== + JSON.stringify(canonical({ 'asia-east2': '10.42.1.0/24' })) + ) { + throw new Error('Asia subnet must be committed before topology planning') + } + const additions = Object.fromEntries(expected.map((cellId) => { + const hostname = cellId.split('-').at(-1) + return [cellId, { + hostname, + region: 'asia-east2', + zone: shape.cells[cellId], + machine_type: 'e2-standard-4', + boot_disk_gb: 30, + boot_image: BOOT_IMAGE, + capacity_requests: 6_000, + database_pool_max: 10, + image, + initially_enabled: false, + connection_hard_cap: 3_000, + connection_unobserved_bound: 60 + }] + })) + for (const cellId of expected) { + if (!existingCells[cellId]) { + throw new Error('Asia cells must be committed before topology planning') + } + if ( + JSON.stringify(canonical(existingCells[cellId])) !== + JSON.stringify(canonical(additions[cellId])) + ) { + throw new Error('committed Asia cell differs from the reviewed topology') + } + } + return { + relay_gce_additional_region_subnetwork_cidrs: existingAdditionalRegions, + relay_gce_cells: existingCells + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const values = argumentsFrom(process.argv.slice(2)) + const existing = JSON.parse(readFileSync(values['existing-json'], 'utf8')) + prepareRelayAsiaTopologyInput({ + existingCells: existing.relay_gce_cells, + existingAdditionalRegions: existing.relay_gce_additional_region_subnetwork_cidrs, + environment: values.environment, + cellIds: values['cell-ids'], + image: values.image + }) +} diff --git a/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs b/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs new file mode 100644 index 00000000000..03622ffbf56 --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs @@ -0,0 +1,87 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { prepareRelayAsiaTopologyInput } from './prepare-relay-asia-topology-input.mjs' + +const image = `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}` + +const additionalRegions = { 'asia-east2': '10.42.1.0/24' } + +const productionCells = () => Object.fromEntries([ + [27, 'asia-east2-a'], + [28, 'asia-east2-b'], + [29, 'asia-east2-c'] +].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, { + hostname: `c${ordinal}`, region: 'asia-east2', zone, + machine_type: 'e2-standard-4', boot_disk_gb: 30, + boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21', + capacity_requests: 6_000, database_pool_max: 10, image, initially_enabled: false, + connection_hard_cap: 3_000, connection_unobserved_bound: 60 + }])) + +test('accepts the exact production topology only after it is durably committed', () => { + const existing = { + 'production-gce-c26': { hostname: 'c26', image: 'existing' }, + ...productionCells() + } + const result = prepareRelayAsiaTopologyInput({ existingCells: existing, + existingAdditionalRegions: additionalRegions, environment: 'production', + cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image }) + assert.equal(result.relay_gce_cells, existing) + assert.equal(result.relay_gce_additional_region_subnetwork_cidrs, additionalRegions) + assert.deepEqual(existing['production-gce-c27'], { + hostname: 'c27', region: 'asia-east2', zone: 'asia-east2-a', + machine_type: 'e2-standard-4', boot_disk_gb: 30, + boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21', + capacity_requests: 6_000, database_pool_max: 10, image, initially_enabled: false, + connection_hard_cap: 3_000, connection_unobserved_bound: 60 + }) +}) + +test('accepts the one exact committed staging Asia cell', () => { + const stagingImage = image.replace('onorca-cloud/', 'onorca-cloud-staging/') + const stagingCell = { + hostname: 'c4', region: 'asia-east2', zone: 'asia-east2-a', + machine_type: 'e2-standard-4', boot_disk_gb: 30, + boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21', + capacity_requests: 6_000, database_pool_max: 10, image: stagingImage, + initially_enabled: false, connection_hard_cap: 3_000, + connection_unobserved_bound: 60 + } + const result = prepareRelayAsiaTopologyInput({ + existingCells: { 'staging-gce-c3': { hostname: 'c3' }, 'staging-gce-c4': stagingCell }, + existingAdditionalRegions: additionalRegions, + environment: 'staging', + cellIds: 'staging-gce-c4', + image: stagingImage + }) + assert.equal(result.relay_gce_cells['staging-gce-c4'].zone, 'asia-east2-a') + assert.equal(result.relay_gce_cells['staging-gce-c4'].image, stagingImage) +}) + +test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drift', () => { + assert.throws(() => prepareRelayAsiaTopologyInput({ + existingCells: productionCells(), existingAdditionalRegions: additionalRegions, + environment: 'production', cellIds: 'production-gce-c27', image + }), /cell IDs/) + assert.throws(() => prepareRelayAsiaTopologyInput({ + existingCells: productionCells(), existingAdditionalRegions: additionalRegions, + environment: 'production', + cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', + image: image.replace('onorca-cloud/', 'other-project/') + }), /environment Relay image/) + assert.throws(() => prepareRelayAsiaTopologyInput({ + existingCells: productionCells(), existingAdditionalRegions: {}, environment: 'production', + cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image + }), /subnet must be committed/) + const missing = productionCells() + delete missing['production-gce-c29'] + assert.throws(() => prepareRelayAsiaTopologyInput({ + existingCells: missing, existingAdditionalRegions: additionalRegions, environment: 'production', + cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image + }), /cells must be committed/) + assert.throws(() => prepareRelayAsiaTopologyInput({ + existingCells: { ...productionCells(), 'production-gce-c27': {} }, + existingAdditionalRegions: additionalRegions, environment: 'production', + cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image + }), /differs from the reviewed topology/) +}) diff --git a/cloud/dev/scripts/prepare-relay-capacity-canary.mjs b/cloud/dev/scripts/prepare-relay-capacity-canary.mjs new file mode 100644 index 00000000000..7bb7574d302 --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-capacity-canary.mjs @@ -0,0 +1,183 @@ +import { pathToFileURL } from 'node:url' +import { + applyExactAdmissionSelector, + inspectAdmissionSelector, + membershipWithStates, + selectorCellState +} from './relay-admission-selector.mjs' + +function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['director-origin', 'cell-id', 'mode']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + const origin = new URL(values['director-origin']) + if (origin.protocol !== 'https:' || origin.origin !== values['director-origin']) { + throw new Error('--director-origin must be a canonical HTTPS origin') + } + if (!['isolate', 'activate', 'restore-fallback', 'restore'].includes(values.mode)) { + throw new Error('--mode must be isolate, activate, restore-fallback, or restore') + } + const cellOrigin = values['cell-origin'] ? new URL(values['cell-origin']) : null + if ( + values.mode === 'isolate' && + (!cellOrigin || cellOrigin.protocol !== 'https:' || cellOrigin.origin !== values['cell-origin']) + ) { + throw new Error('--cell-origin must be a canonical HTTPS origin for isolate mode') + } + const restoreGeneralCellIds = values['general-cell-ids']?.split(',').filter(Boolean) ?? [] + if ( + ['restore-fallback', 'restore'].includes(values.mode) && + restoreGeneralCellIds.length === 0 + ) { + throw new Error('--general-cell-ids is required for restore modes') + } + if (values.mode === 'restore' && !restoreGeneralCellIds.includes(values['cell-id'])) { + throw new Error('--general-cell-ids must include the canary for restore mode') + } + if (values.mode === 'restore-fallback' && restoreGeneralCellIds.includes(values['cell-id'])) { + throw new Error('--general-cell-ids cannot include the canary for fallback restore') + } + if (new Set(restoreGeneralCellIds).size !== restoreGeneralCellIds.length) { + throw new Error('--general-cell-ids must be distinct') + } + return { + directorOrigin: origin.origin, + cellOrigin: cellOrigin?.origin, + cellId: values['cell-id'], + mode: values.mode, + restoreGeneralCellIds + } +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({})) + if (!response.ok) throw new Error(`${label} returned ${response.status}`) + return body +} + +function sameMembership(left, right) { + return JSON.stringify(left) === JSON.stringify(right) +} + +async function legacyCellState(post, cellId) { + const result = await post('/v1/admin/cell-status', { v: 1, cellId }) + if (result.status?.cellId !== cellId) throw new Error('legacy admission status is invalid') + const state = result.status.admissionState + if (!['existing-only', 'migration-only', 'general'].includes(state)) { + throw new Error('legacy admission status is invalid') + } + return state +} + +async function applyLegacyStates(post, before, states, order) { + const expected = membershipWithStates(before.selector, states) + let changed = false + for (const cellId of order) { + const desired = states[cellId] + const current = await legacyCellState(post, cellId) + if (current === 'existing-only' && desired !== current) { + throw new Error(`legacy admission cannot re-enable existing-only cell ${cellId}`) + } + if (current === desired) continue + let cause + try { + await post('/v1/admin/cell-state', { v: 1, cellId, state: desired }) + } catch (error) { + cause = error + } + if ((await legacyCellState(post, cellId)) !== desired) { + const detail = cause instanceof Error ? `: ${cause.message}` : '' + throw new Error(`legacy admission did not commit ${cellId} exactly${detail}`, { cause }) + } + changed = true + } + const verified = await inspectAdmissionSelector(post) + if (verified.selector.generation !== 0 || + !sameMembership(verified.selector.membership, expected)) { + throw new Error('legacy admission membership changed unexpectedly') + } + return { changed, selector: verified.selector } +} + +export async function prepareCapacityCanary(config, overrides = {}) { + const fetchImpl = overrides.fetch ?? fetch + const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN + if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') + const postAt = async (origin, path, body) => + await responseJson( + await fetchImpl(`${origin}${path}`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }), + path + ) + const post = async (path, body) => await postAt(config.directorOrigin, path, body) + const before = await inspectAdmissionSelector(post) + const state = selectorCellState(before.selector, config.cellId) + if (state === 'existing-only' && config.mode !== 'restore-fallback') { + throw new Error('capacity canary cannot restore existing-only admission') + } + const states = + config.mode === 'isolate' + ? { [config.cellId]: 'migration-only' } + : config.mode === 'activate' + ? Object.fromEntries([ + ...before.selector.membership.general.map((cellId) => [ + cellId, + cellId === config.cellId ? 'general' : 'migration-only' + ]), + [config.cellId, 'general'] + ]) + : Object.fromEntries([ + ...config.restoreGeneralCellIds.map((cellId) => [cellId, 'general']), + ...(config.mode === 'restore-fallback' + ? [[config.cellId, state === 'existing-only' ? 'existing-only' : 'migration-only']] + : []) + ]) + const membership = membershipWithStates(before.selector, states) + const legacyOrder = + config.mode === 'activate' + ? [config.cellId, ...before.selector.membership.general.filter((id) => id !== config.cellId)] + : config.mode === 'restore-fallback' + ? [...config.restoreGeneralCellIds, config.cellId] + : Object.keys(states) + const result = before.selector.generation === 0 + ? await applyLegacyStates(post, before, states, legacyOrder) + : sameMembership(membership, before.selector.membership) + ? { changed: false, selector: before.selector } + : await applyExactAdmissionSelector(post, membership, { + expectedCurrentSelector: before.selector + }) + if (config.mode === 'isolate') { + await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 }) + } + return { + changed: result.changed, + generation: result.selector.generation, + ...(config.mode === 'isolate' ? { drained: true } : {}) + } +} + +export async function main(argv = process.argv.slice(2)) { + const config = parseArguments(argv) + const result = await prepareCapacityCanary(config) + process.stdout.write( + `${JSON.stringify({ event: 'relay_capacity_canary_admission', cellId: config.cellId, mode: config.mode, ...result })}\n` + ) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs b/cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs new file mode 100644 index 00000000000..d8e752aa500 --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs @@ -0,0 +1,300 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { relayWorkflowUrl } from './relay-repository.mjs' +import { prepareCapacityCanary } from './prepare-relay-capacity-canary.mjs' + +function harness(initialState, options = {}) { + const { + generation = 4, + ambiguousCellState = false, + rejectCellState = false, + fallbackState = 'general', + extraGeneralCellIds = [] + } = options + let selector = { + generation, + attemptId: 'initial', + membership: { + existingOnly: [ + 'staging-gce-c1', + ...(initialState === 'existing-only' ? ['staging-gce-c3'] : []) + ], + migrationOnly: [ + ...(fallbackState === 'migration-only' ? ['staging-gce-c2'] : []), + ...(initialState === 'migration-only' ? ['staging-gce-c3'] : []) + ], + general: [ + ...extraGeneralCellIds, + ...(fallbackState === 'general' ? ['staging-gce-c2'] : []), + ...(initialState === 'general' ? ['staging-gce-c3'] : []) + ].sort() + } + } + let intent = null + let applies = 0 + let drains = 0 + const cellStateChanges = [] + const fetch = async (url, options) => { + const path = new URL(url).pathname + const body = JSON.parse(options.body) + if (path === '/v1/admin/drain') { + assert.deepEqual(body, { v: 1, graceMs: 0 }) + drains++ + return Response.json({ ok: true }) + } + if (path === '/v1/admin/cell-status') { + const state = selector.membership.existingOnly.includes(body.cellId) + ? 'existing-only' + : selector.membership.migrationOnly.includes(body.cellId) + ? 'migration-only' + : 'general' + return Response.json({ status: { cellId: body.cellId, admissionState: state } }) + } + if (path === '/v1/admin/cell-state') { + assert.equal(selector.generation, 0) + if (rejectCellState) { + return Response.json({ error: 'invalid_token' }, { status: 401 }) + } + const keys = { + 'existing-only': 'existingOnly', + 'migration-only': 'migrationOnly', + general: 'general' + } + for (const cells of Object.values(selector.membership)) { + const index = cells.indexOf(body.cellId) + if (index !== -1) cells.splice(index, 1) + } + selector.membership[keys[body.state]].push(body.cellId) + for (const cells of Object.values(selector.membership)) cells.sort() + cellStateChanges.push({ cellId: body.cellId, state: body.state }) + if (ambiguousCellState) throw new Error('response lost') + return Response.json({ ok: true }) + } + if (path.endsWith('/status')) return Response.json({ selector, intent }) + applies++ + selector = { + generation: body.expectedGeneration + 1, + attemptId: body.attemptId, + membership: body.membership + } + intent = { + attemptId: body.attemptId, + expectedGeneration: body.expectedGeneration, + intendedGeneration: selector.generation, + membership: selector.membership, + state: 'committed' + } + return Response.json({ changed: true, selector }) + } + return { + fetch, + selector: () => selector, + applies: () => applies, + drains: () => drains, + cellStateChanges + } +} + +const config = { + directorOrigin: 'https://relay.example.com', + cellOrigin: 'https://c3.relay.example.com', + cellId: 'staging-gce-c3', + mode: 'isolate', + restoreGeneralCellIds: [] +} + +test('isolates a general canary as migration-only', async () => { + const testHarness = harness('general') + assert.deepEqual( + await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }), + { changed: true, generation: 5, drained: true } + ) + assert.deepEqual(testHarness.selector().membership.migrationOnly, [config.cellId]) + assert.equal(testHarness.applies(), 1) + assert.equal(testHarness.drains(), 1) +}) + +test('activates the canary as the only general cell', async () => { + const testHarness = harness('migration-only') + assert.deepEqual( + await prepareCapacityCanary( + { ...config, mode: 'activate' }, + { fetch: testHarness.fetch, token: 'masked' } + ), + { changed: true, generation: 5 } + ) + assert.deepEqual(testHarness.selector().membership, { + existingOnly: ['staging-gce-c1'], + migrationOnly: ['staging-gce-c2'], + general: [config.cellId] + }) + assert.equal(testHarness.applies(), 1) +}) + +test('restores the reviewed staging general membership', async () => { + const testHarness = harness('migration-only') + assert.deepEqual( + await prepareCapacityCanary( + { + ...config, + mode: 'restore', + restoreGeneralCellIds: ['staging-gce-c2', config.cellId] + }, + { fetch: testHarness.fetch, token: 'masked' } + ), + { changed: true, generation: 5 } + ) + assert.deepEqual(testHarness.selector().membership, { + existingOnly: ['staging-gce-c1'], + migrationOnly: [], + general: ['staging-gce-c2', config.cellId] + }) +}) + +test('restores the fallback without promoting a possibly drained canary', async () => { + const testHarness = harness('general') + await prepareCapacityCanary( + { + ...config, + mode: 'restore-fallback', + restoreGeneralCellIds: ['staging-gce-c2'] + }, + { fetch: testHarness.fetch, token: 'masked' } + ) + assert.deepEqual(testHarness.selector().membership, { + existingOnly: ['staging-gce-c1'], + migrationOnly: [config.cellId], + general: ['staging-gce-c2'] + }) +}) + +test('restores the fallback while preserving an irreversible canary', async () => { + const testHarness = harness('existing-only') + await prepareCapacityCanary( + { + ...config, + mode: 'restore-fallback', + restoreGeneralCellIds: ['staging-gce-c2'] + }, + { fetch: testHarness.fetch, token: 'masked' } + ) + assert.deepEqual(testHarness.selector().membership, { + existingOnly: ['staging-gce-c1', config.cellId], + migrationOnly: [], + general: ['staging-gce-c2'] + }) +}) + +test('uses exact legacy admission writes before the selector boundary', async () => { + const testHarness = harness('general', { generation: 0 }) + assert.deepEqual( + await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }), + { changed: true, generation: 0, drained: true } + ) + assert.deepEqual(testHarness.cellStateChanges, [ + { cellId: config.cellId, state: 'migration-only' } + ]) + assert.equal(testHarness.drains(), 1) +}) + +test('promotes a legacy canary before demoting its fallback', async () => { + const testHarness = harness('migration-only', { generation: 0 }) + await prepareCapacityCanary( + { ...config, mode: 'activate' }, + { fetch: testHarness.fetch, token: 'masked' } + ) + assert.deepEqual(testHarness.cellStateChanges, [ + { cellId: config.cellId, state: 'general' }, + { cellId: 'staging-gce-c2', state: 'migration-only' } + ]) +}) + +test('makes the canary sole general with the live legacy membership shape', async () => { + const extraGeneralCellIds = ['combined', 'staging-c1', 'staging-c2'] + const testHarness = harness('migration-only', { generation: 0, extraGeneralCellIds }) + const activate = { ...config, mode: 'activate' } + await prepareCapacityCanary(activate, { fetch: testHarness.fetch, token: 'masked' }) + assert.deepEqual(testHarness.cellStateChanges, [ + { cellId: config.cellId, state: 'general' }, + { cellId: 'combined', state: 'migration-only' }, + { cellId: 'staging-c1', state: 'migration-only' }, + { cellId: 'staging-c2', state: 'migration-only' }, + { cellId: 'staging-gce-c2', state: 'migration-only' } + ]) +}) + +test('restores a legacy fallback before demoting the target', async () => { + const testHarness = harness('general', { + generation: 0, + fallbackState: 'migration-only' + }) + await prepareCapacityCanary( + { + ...config, + mode: 'restore-fallback', + restoreGeneralCellIds: ['staging-gce-c2'] + }, + { fetch: testHarness.fetch, token: 'masked' } + ) + assert.deepEqual(testHarness.cellStateChanges, [ + { cellId: 'staging-gce-c2', state: 'general' }, + { cellId: config.cellId, state: 'migration-only' } + ]) +}) + +test('keeps an already restored legacy fallback unchanged', async () => { + const testHarness = harness('migration-only', { generation: 0 }) + assert.deepEqual( + await prepareCapacityCanary( + { + ...config, + mode: 'restore-fallback', + restoreGeneralCellIds: ['staging-gce-c2'] + }, + { fetch: testHarness.fetch, token: 'masked' } + ), + { changed: false, generation: 0 } + ) + assert.deepEqual(testHarness.cellStateChanges, []) + assert.deepEqual(testHarness.selector().membership, { + existingOnly: ['staging-gce-c1'], + migrationOnly: [config.cellId], + general: ['staging-gce-c2'] + }) +}) + +test('recovers an ambiguous legacy admission response by exact readback', async () => { + const testHarness = harness('general', { generation: 0, ambiguousCellState: true }) + await assert.doesNotReject( + prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }) + ) + assert.equal(testHarness.drains(), 1) +}) + +test('reports a rejected legacy admission write without draining', async () => { + const testHarness = harness('general', { generation: 0, rejectCellState: true }) + const operation = prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }) + await assert.rejects(operation, /cell-state returned 401/) + assert.equal(testHarness.drains(), 0) +}) + +test('the staging workflow supplies every required capacity transition argument', () => { + const workflow = readFileSync( + relayWorkflowUrl('prove-relay-staging-capacity.yml'), + 'utf8' + ) + const verifyCalls = workflow.match( + /node dev\/scripts\/verify-relay-capacity-transition\.mjs[\s\S]*?(?=\n\s*\n|\n\s*- name:)/g + ) + assert.ok(verifyCalls?.length >= 5) + for (const call of verifyCalls) { + for (const flag of ['--cell-origin', '--heartbeat', '--admission', '--draining', '--activity']) { + assert.match(call, new RegExp(flag)) + } + } + const isolate = workflow.match( + /node dev\/scripts\/prepare-relay-capacity-canary\.mjs[\s\S]*?--mode isolate/ + )?.[0] + assert.match(isolate, /--cell-origin/) +}) diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs new file mode 100644 index 00000000000..5791c9f20e6 --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs @@ -0,0 +1,116 @@ +import { pathToFileURL } from 'node:url' +import { + applyExactAdmissionSelector, + inspectAdmissionSelector, + membershipWithStates, + selectorCellState +} from './relay-admission-selector.mjs' + +const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' +export const PRODUCTION_CAPACITY_CELL_IDS = [ + 'production-gce-c7', + 'production-gce-c8', + 'production-gce-c9', + 'production-gce-c10', + 'production-gce-c13', + 'production-gce-c14', + 'production-gce-c15', + 'production-gce-c16', + 'production-gce-c19', + 'production-gce-c20', + 'production-gce-c21', + 'production-gce-c22', + 'production-gce-c23', + 'production-gce-c24', + 'production-gce-c25', + 'production-gce-c26' +] + +function cellOrigin(cellId) { + return `https://${cellId.slice('production-gce-'.length)}.relay.onorca.dev` +} + +export function parseProductionCapacityCellArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + if (!['isolate', 'drain', 'activate'].includes(values.mode)) { + throw new Error('--mode must be isolate, drain, or activate') + } + const cellId = values['cell-id'] + if (!PRODUCTION_CAPACITY_CELL_IDS.includes(cellId)) { + throw new Error('production capacity target is not approved') + } + const expectedCellOrigin = cellOrigin(cellId) + if ( + values['director-origin'] !== DIRECTOR_ORIGIN || + values['cell-origin'] !== expectedCellOrigin + ) { + throw new Error('production capacity target origin is not exact') + } + return { + directorOrigin: DIRECTOR_ORIGIN, + cellOrigin: expectedCellOrigin, + cellId, + mode: values.mode + } +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({})) + if (!response.ok) throw new Error(`${label} returned ${response.status}`) + return body +} + +export async function prepareProductionCapacityCell(config, overrides = {}) { + const fetchImpl = overrides.fetch ?? fetch + const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN + if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') + const postAt = async (origin, path, body) => + await responseJson( + await fetchImpl(`${origin}${path}`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }), + path + ) + const post = async (path, body) => await postAt(config.directorOrigin, path, body) + if (config.mode === 'drain') { + await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 }) + return { changed: false, drained: true } + } + const before = await inspectAdmissionSelector(post) + const state = selectorCellState(before.selector, config.cellId) + if (state === 'existing-only') throw new Error('production capacity target is irreversible') + const desiredState = config.mode === 'isolate' ? 'migration-only' : 'general' + const membership = membershipWithStates(before.selector, { [config.cellId]: desiredState }) + const result = await applyExactAdmissionSelector(post, membership, { + expectedCurrentSelector: before.selector + }) + return { + changed: result.changed, + generation: result.selector.generation, + admissionState: desiredState + } +} + +export async function main(argv = process.argv.slice(2)) { + const config = parseProductionCapacityCellArguments(argv) + const result = await prepareProductionCapacityCell(config) + process.stdout.write( + `${JSON.stringify({ event: 'relay_production_capacity_canary', cellId: config.cellId, mode: config.mode, ...result })}\n` + ) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs new file mode 100644 index 00000000000..c5d0a9db3bc --- /dev/null +++ b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs @@ -0,0 +1,173 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { + parseProductionCapacityCellArguments, + prepareProductionCapacityCell, + PRODUCTION_CAPACITY_CELL_IDS +} from './prepare-relay-production-capacity-canary.mjs' + +const config = { + directorOrigin: 'https://relay.onorca.dev', + cellOrigin: 'https://c26.relay.onorca.dev', + cellId: 'production-gce-c26' +} + +const membership = { + existingOnly: ['production-gce-c1'], + migrationOnly: ['production-gce-c17'], + general: ['production-gce-c25', 'production-gce-c26'] +} + +function response(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +function canaryFetch() { + let selector = { generation: 20, attemptId: null, membership } + const calls = [] + const fetch = async (url, init) => { + const path = new URL(url).pathname + const body = JSON.parse(init.body) + calls.push({ path, body }) + if (path === '/v1/admin/admission-selector/status') { + return response({ + v: 1, + selector, + intent: body.attemptId + ? { + attemptId: body.attemptId, + state: 'committed', + expectedGeneration: selector.generation - 1, + intendedGeneration: selector.generation, + membership: selector.membership + } + : null + }) + } + if (path === '/v1/admin/admission-selector/apply') { + selector = { + generation: selector.generation + 1, + attemptId: body.attemptId, + membership: body.membership + } + return response({ v: 1, changed: true, selector }) + } + if (path === '/v1/admin/drain') return response({ v: 1, draining: true }) + throw new Error(`unexpected ${path}`) + } + return { calls, fetch, selector: () => selector } +} + +describe('production Relay capacity cell admission', () => { + it('allows only the serving rollout cells', () => { + assert.deepEqual(PRODUCTION_CAPACITY_CELL_IDS, [ + 'production-gce-c7', + 'production-gce-c8', + 'production-gce-c9', + 'production-gce-c10', + 'production-gce-c13', + 'production-gce-c14', + 'production-gce-c15', + 'production-gce-c16', + 'production-gce-c19', + 'production-gce-c20', + 'production-gce-c21', + 'production-gce-c22', + 'production-gce-c23', + 'production-gce-c24', + 'production-gce-c25', + 'production-gce-c26' + ]) + assert.deepEqual(parseProductionCapacityCellArguments([ + '--director-origin', 'https://relay.onorca.dev', + '--cell-origin', 'https://c7.relay.onorca.dev', + '--cell-id', 'production-gce-c7', + '--mode', 'isolate' + ]), { + directorOrigin: 'https://relay.onorca.dev', + cellOrigin: 'https://c7.relay.onorca.dev', + cellId: 'production-gce-c7', + mode: 'isolate' + }) + assert.throws(() => parseProductionCapacityCellArguments([ + '--director-origin', 'https://relay.onorca.dev', + '--cell-origin', 'https://c17.relay.onorca.dev', + '--cell-id', 'production-gce-c17', + '--mode', 'isolate' + ]), /not approved/) + assert.throws(() => parseProductionCapacityCellArguments([ + '--director-origin', 'https://relay.onorca.dev', + '--cell-origin', 'https://c8.relay.onorca.dev', + '--cell-id', 'production-gce-c7', + '--mode', 'isolate' + ]), /origin is not exact/) + }) + + it('isolates only the selected cell without depending on its runtime', async () => { + const fake = canaryFetch() + const result = await prepareProductionCapacityCell( + { ...config, mode: 'isolate' }, + { fetch: fake.fetch, token: 'token' } + ) + assert.equal(result.admissionState, 'migration-only') + assert.deepEqual(fake.selector().membership, { + existingOnly: ['production-gce-c1'], + migrationOnly: ['production-gce-c17', 'production-gce-c26'], + general: ['production-gce-c25'] + }) + assert.doesNotMatch(fake.calls.map(({ path }) => path).join(','), /\/v1\/admin\/drain/) + }) + + it('drains the selected cell independently after durable isolation', async () => { + const fake = canaryFetch() + const result = await prepareProductionCapacityCell( + { ...config, mode: 'drain' }, + { fetch: fake.fetch, token: 'token' } + ) + assert.deepEqual(result, { changed: false, drained: true }) + assert.deepEqual(fake.calls, [{ + path: '/v1/admin/drain', + body: { v: 1, graceMs: 0 } + }]) + }) + + it('restores only the selected cell to general admission', async () => { + const fake = canaryFetch() + await prepareProductionCapacityCell( + { ...config, mode: 'isolate' }, + { fetch: fake.fetch, token: 'token' } + ) + const result = await prepareProductionCapacityCell( + { ...config, mode: 'activate' }, + { fetch: fake.fetch, token: 'token' } + ) + assert.equal(result.admissionState, 'general') + assert.deepEqual(fake.selector().membership, membership) + }) + + it('refuses an irreversible existing-only target', async () => { + const fetch = async () => response({ + v: 1, + selector: { + generation: 20, + attemptId: null, + membership: { + existingOnly: ['production-gce-c26'], + migrationOnly: ['production-gce-c17'], + general: ['production-gce-c25'] + } + }, + intent: null + }) + await assert.rejects( + prepareProductionCapacityCell( + { ...config, mode: 'isolate' }, + { fetch, token: 'token' } + ), + /irreversible/ + ) + }) +}) diff --git a/cloud/dev/scripts/probe-relay-legacy-admission.mjs b/cloud/dev/scripts/probe-relay-legacy-admission.mjs new file mode 100644 index 00000000000..3529c7d70ac --- /dev/null +++ b/cloud/dev/scripts/probe-relay-legacy-admission.mjs @@ -0,0 +1,73 @@ +import { randomBytes } from 'node:crypto' +import { pathToFileURL } from 'node:url' + +const WRONG_CELL = 4409 +const DRAINING = 4503 + +function once(socket, event, listener) { + if (typeof socket.once === 'function') { + socket.once(event, listener) + return + } + if (typeof socket.addEventListener !== 'function') { + throw new Error('WebSocket event API is unavailable') + } + socket.addEventListener(event, (value) => { + if (event === 'close') listener(value.code) + else if (event === 'error') listener(value.error ?? new Error(value.message)) + else listener() + }, { once: true }) +} + +export function parseLegacyAdmissionProbeArguments(argv) { + if (argv.length !== 2 || argv[0] !== '--cell-origin') throw new Error('invalid arguments') + const origin = new URL(argv[1]) + if (origin.protocol !== 'https:' || origin.origin !== argv[1]) { + throw new Error('--cell-origin must be a canonical HTTPS origin') + } + return { cellOrigin: origin.origin } +} + +export async function probeLegacyAdmission(config, overrides = {}) { + const Socket = overrides.WebSocket ?? globalThis.WebSocket + if (typeof Socket !== 'function') throw new Error('WebSocket is unavailable') + const random = overrides.randomBytes ?? randomBytes + const timeoutMs = overrides.timeoutMs ?? 15_000 + const hostId = random(12).toString('base64url') + const credential = random(32).toString('base64url') + const url = `${config.cellOrigin.replace('https://', 'wss://')}/v1/connect/${hostId}` + await new Promise((resolve, reject) => { + const socket = new Socket(url) + const timer = setTimeout(() => { + if (typeof socket.terminate === 'function') socket.terminate() + else socket.close() + reject(new Error('legacy admission probe timed out')) + }, timeoutMs) + once(socket, 'open', () => { + socket.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential })) + }) + once(socket, 'close', (code) => { + clearTimeout(timer) + if (code === WRONG_CELL) resolve() + else if (code === DRAINING) reject(new Error('legacy cell is draining')) + else reject(new Error(`legacy admission probe closed with ${code}`)) + }) + once(socket, 'error', (error) => { + clearTimeout(timer) + reject(new Error(`legacy admission probe failed: ${error.message}`)) + }) + }) + return { accepting: true } +} + +export async function main(argv = process.argv.slice(2)) { + const result = await probeLegacyAdmission(parseLegacyAdmissionProbeArguments(argv)) + process.stdout.write(`${JSON.stringify({ event: 'relay_legacy_admission_verified', ...result })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/probe-relay-legacy-admission.test.mjs b/cloud/dev/scripts/probe-relay-legacy-admission.test.mjs new file mode 100644 index 00000000000..7c0d078b4f0 --- /dev/null +++ b/cloud/dev/scripts/probe-relay-legacy-admission.test.mjs @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' +import { test } from 'node:test' +import { + parseLegacyAdmissionProbeArguments, + probeLegacyAdmission +} from './probe-relay-legacy-admission.mjs' + +function socketClosingWith(code, observed) { + return class extends EventEmitter { + constructor(url) { + super() + observed.url = url + queueMicrotask(() => this.emit('open')) + } + + send(payload) { + observed.payload = JSON.parse(payload) + queueMicrotask(() => this.emit('close', code)) + } + + terminate() {} + } +} + +function nativeSocketClosingWith(code) { + return class extends EventTarget { + constructor() { + super() + queueMicrotask(() => this.dispatchEvent(new Event('open'))) + } + + send() { + const event = new Event('close') + Object.defineProperty(event, 'code', { value: code }) + queueMicrotask(() => this.dispatchEvent(event)) + } + + close() {} + } +} + +const config = { cellOrigin: 'https://c2.relay.example.com' } +const random = (length) => Buffer.alloc(length, length) + +test('accepts only a canonical cell origin', () => { + assert.deepEqual( + parseLegacyAdmissionProbeArguments(['--cell-origin', config.cellOrigin]), + config + ) + assert.throws( + () => parseLegacyAdmissionProbeArguments(['--cell-origin', `${config.cellOrigin}/path`]), + /canonical/ + ) +}) + +test('proves admission with a synthetic invalid credential and exposes no identifier', async () => { + const observed = {} + assert.deepEqual( + await probeLegacyAdmission(config, { + WebSocket: socketClosingWith(4409, observed), + randomBytes: random + }), + { accepting: true } + ) + assert.match(observed.url, /^wss:\/\/c2\.relay\.example\.com\/v1\/connect\/[A-Za-z0-9_-]{16}$/) + assert.deepEqual(Object.keys(observed.payload).sort(), ['credential', 'mode', 'type', 'v']) +}) + +test('uses the dependency-free Node WebSocket event API', async () => { + await assert.doesNotReject( + probeLegacyAdmission(config, { + WebSocket: nativeSocketClosingWith(4409), + randomBytes: random + }) + ) +}) + +test('rejects the legacy draining close and any unknown outcome', async () => { + for (const [code, message] of [[4503, /draining/], [4401, /closed with 4401/]]) { + await assert.rejects( + probeLegacyAdmission(config, { + WebSocket: socketClosingWith(code, {}), + randomBytes: random + }), + message + ) + } +}) diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs new file mode 100644 index 00000000000..7500d8bd14c --- /dev/null +++ b/cloud/dev/scripts/probe-relay-rehome-trust.mjs @@ -0,0 +1,80 @@ +import { pathToFileURL } from 'node:url' + +const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/ +const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' + +export function parseRehomeTrustProbeArguments(argv, environment = process.env) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['director-origin', 'cell-id', 'cell-incarnation']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (values['director-origin'] !== DIRECTOR_ORIGIN) { + throw new Error('--director-origin must be the production Relay origin') + } + if (!PRODUCTION_CELL.test(values['cell-id'])) throw new Error('--cell-id is not approved') + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + values['cell-incarnation'] + )) throw new Error('--cell-incarnation is invalid') + const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN + if (!token || token.length > 8_192 || !/^[^.]+\.[^.]+\.[^.]+$/.test(token)) { + throw new Error('admin identity token is unavailable') + } + return { + directorOrigin: DIRECTOR_ORIGIN, + cellId: values['cell-id'], + cellIncarnation: values['cell-incarnation'], + token + } +} + +export async function probeRehomeTrust(config, dependencies = {}) { + const fetchImpl = dependencies.fetch ?? fetch + const response = await fetchImpl( + `${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`, + { + method: 'POST', + headers: { + authorization: `Bearer ${config.token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + v: 1, + sourceCellId: config.cellId, + sourceCellIncarnation: config.cellIncarnation + }), + signal: AbortSignal.timeout(30_000) + } + ) + const body = await response.json().catch(() => ({})) + if (!response.ok) { + throw new Error(`application-mediated rehome trust probe returned ${response.status}`) + } + if ( + body.v !== 1 || + body.dedicatedIdentity?.firstOutcome !== 'host-not-connected' || + body.dedicatedIdentity?.secondOutcome !== 'host-not-connected' || + body.dedicatedIdentity?.accepted !== true || + body.dedicatedIdentity?.idempotent !== true || + body.sharedRuntimeIdentityRejected !== true || + body.proven !== true + ) throw new Error('application-mediated rehome trust proof is incomplete') + return body +} + +export async function main(argv = process.argv.slice(2)) { + const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv)) + process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_trust_verified', ...result })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs new file mode 100644 index 00000000000..509e9d53c7d --- /dev/null +++ b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + parseRehomeTrustProbeArguments, + probeRehomeTrust +} from './probe-relay-rehome-trust.mjs' + +const argv = [ + '--director-origin', 'https://relay.onorca.dev', + '--cell-id', 'production-gce-c7', + '--cell-incarnation', '11111111-1111-4111-8111-111111111111' +] +const environment = { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' } + +test('binds the application-mediated probe to an exact approved cell incarnation', () => { + assert.equal(parseRehomeTrustProbeArguments(argv, environment).cellId, 'production-gce-c7') + assert.throws(() => parseRehomeTrustProbeArguments( + argv.with(1, 'https://other.example.test'), + environment + )) + assert.throws(() => parseRehomeTrustProbeArguments(argv, { + ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-token' + })) +}) + +test('requires complete aggregate application-mediated trust proof', async () => { + const config = parseRehomeTrustProbeArguments(argv, environment) + const result = await probeRehomeTrust(config, { + fetch: async (url, init) => { + assert.equal(url, 'https://relay.onorca.dev/v1/admin/regional-rehome-trust-probe') + assert.deepEqual(JSON.parse(init.body), { + v: 1, + sourceCellId: 'production-gce-c7', + sourceCellIncarnation: '11111111-1111-4111-8111-111111111111' + }) + return Response.json({ + v: 1, + dedicatedIdentity: { + firstOutcome: 'host-not-connected', + secondOutcome: 'host-not-connected', + accepted: true, + idempotent: true + }, + sharedRuntimeIdentityRejected: true, + proven: true + }) + } + }) + assert.equal(result.proven, true) +}) + +test('rejects partial or mismatched proof', async () => { + const config = parseRehomeTrustProbeArguments(argv, environment) + await assert.rejects( + probeRehomeTrust(config, { + fetch: async () => Response.json({ + v: 1, + dedicatedIdentity: { + firstOutcome: 'host-not-connected', + secondOutcome: 'host-not-connected', + accepted: true, + idempotent: true + }, + sharedRuntimeIdentityRejected: false, + proven: false + }) + }), + /incomplete/ + ) +}) diff --git a/cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs b/cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs new file mode 100644 index 00000000000..612d48abdcd --- /dev/null +++ b/cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs' +import { readRelayWorkflow } from './relay-repository.mjs' + +const production = source('infra/terraform/environments/production.tfvars') +const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml') +const jobWorkflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml') + +const RELAY_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay' + +function source(path) { + return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8') +} + +// Slice each "" = { ... } entry out of relay_gce_cells. +function productionCells() { + const block = production.slice(production.indexOf('relay_gce_cells = {')) + const cells = new Map() + for (const match of block.matchAll(/"(production-gce-c\d+)" = \{([\s\S]*?)\n {2}\}/g)) { + cells.set(match[1], match[2]) + } + assert.ok(cells.size > 0, 'relay_gce_cells parsed empty') + return cells +} + +function hardCap(body) { + const match = body.match(/connection_hard_cap\s*=\s*(\d+)/) + return match ? Number(match[1]) : undefined +} + +function imageDigest(body) { + const match = body.match(/^\s*image\s*=\s*"([^"]+)"/m) + assert.ok(match, 'cell entry has no image') + const [repository, digest] = match[1].split('@') + assert.equal(repository, RELAY_REPOSITORY) + assert.match(digest, /^sha256:[0-9a-f]{64}$/) + return digest +} + +function workflowPin(workflow, name) { + const match = workflow.match(new RegExp(`${name}: (sha256:[0-9a-f]{64})`)) + assert.ok(match, `${name} is missing or not a full digest`) + return match[1] +} + +test('every production cell pins a full relay image digest', () => { + for (const [cellId, body] of productionCells()) { + assert.match(imageDigest(body), /^sha256:[0-9a-f]{64}$/, `${cellId} image digest`) + } +}) + +test('the 1,000-cap cells are exactly the canonical capacity set', () => { + const thousandCap = [...productionCells()] + .filter(([, body]) => hardCap(body) === 1000) + .map(([cellId]) => cellId) + assert.deepEqual([...thousandCap].sort(), [...PRODUCTION_CAPACITY_CELL_IDS].sort()) +}) + +test('the 1,000-cap cells all serve one image digest', () => { + const digests = new Map() + for (const [cellId, body] of productionCells()) { + if (hardCap(body) !== 1000) continue + const digest = imageDigest(body) + if (!digests.has(digest)) digests.set(digest, []) + digests.get(digest).push(cellId) + } + assert.equal( + digests.size, + 1, + `1,000-cap cells split across digests: ${JSON.stringify([...digests])}` + ) + assert.equal([...digests.values()][0].length, PRODUCTION_CAPACITY_CELL_IDS.length) +}) + +// COMPATIBLE_CELL_IMAGE_DIGEST is one half of a reviewed (director, cell) skew pair, not a +// claim about what the fleet serves; it is re-derived by hand for each capacity wave. So it +// is deliberately NOT tied to the tfvars digest — only to its twin in the dispatch workflow. +test('both capacity workflows declare the same reviewed image pins', () => { + for (const name of ['PREDECESSOR_IMAGE_DIGEST', 'COMPATIBLE_CELL_IMAGE_DIGEST']) { + assert.equal(workflowPin(dispatchWorkflow, name), workflowPin(jobWorkflow, name), name) + } + workflowPin(jobWorkflow, 'COMPATIBLE_DIRECTOR_IMAGE_DIGEST') +}) diff --git a/cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs b/cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs new file mode 100644 index 00000000000..d79d4f91046 --- /dev/null +++ b/cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { + LEASED_WORKFLOWS, + LOCK_GROUPS, + NOT_A_CLOUD_SQL_CANDIDATE, + PRODUCTION_LEASE, + SELECTABLE_LEASE, + STAGING_LEASE, + concurrencyBlocks, + entrypointsFor, + jobIf, + jobNeeds, + jobs, + leaseSteps, + leaseStepsByJob, + mutatesSharedInstance, + readWorkflow, + reusableCalls, + revisionMintingScripts, + workflowFiles +} from './cloud-sql-rollout-lock-census.mjs' +import { relayWorkflowFile } from './relay-repository.mjs' + +const expectedLease = { production: PRODUCTION_LEASE, staging: STAGING_LEASE, selectable: SELECTABLE_LEASE } +const leasedFiles = Object.keys(LEASED_WORKFLOWS) + +function contractFiles(file) { + return [file, ...(LEASED_WORKFLOWS[file].leaseFiles ?? [])] +} + +test('every locked workflow declares exactly one lock group that never cancels', () => { + for (const file of leasedFiles) { + const blocks = concurrencyBlocks(readWorkflow(file)) + assert.equal(blocks.length, 1, `${file} must declare exactly one concurrency block`) + assert.equal(blocks[0].group, LEASED_WORKFLOWS[file].group, file) + assert.equal(blocks[0].cancelInProgress, 'false', file) + } +}) + +test('every locked workflow takes the lease for its own environment', () => { + for (const file of leasedFiles) { + const lease = expectedLease[LEASED_WORKFLOWS[file].env] + const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member))) + assert.ok(steps.length > 0, `${file} must use the Cloud SQL rollout lease action`) + for (const step of steps) { + assert.equal(step.bucket, lease.bucket, file) + assert.equal(step.object, lease.object, file) + } + } +}) + +test('the lease step runs after the credential that authorizes it', () => { + for (const file of leasedFiles) { + for (const member of contractFiles(file)) { + const text = readWorkflow(member) + const lines = text.split('\n') + for (const step of leaseSteps(text)) { + const before = lines.slice(0, step.line - 1) + const gcloud = before.lastIndexOf(' - uses: google-github-actions/setup-gcloud@v2') + assert.notEqual(gcloud, -1, `${member}: lease step at line ${step.line} has no setup-gcloud before it`) + assert.ok( + before.lastIndexOf(' - uses: actions/checkout@v4') !== -1, + `${member}: lease step at line ${step.line} runs before the local action is checked out` + ) + } + } + } +}) + +test('multi-wave workflows hold one lease per run and free it exactly once', () => { + for (const file of leasedFiles) { + const entry = LEASED_WORKFLOWS[file] + const waveFiles = entry.leaseFiles ?? [] + const callCount = waveFiles.reduce( + (total, member) => total + (reusableCalls(readWorkflow(file)).get(member) ?? 0), + 0 + ) + if (!entry.reentrant) { + assert.ok(callCount <= 1, `${file} calls a leased reusable job ${callCount} times; it needs a release job`) + for (const member of contractFiles(file)) { + for (const step of leaseSteps(readWorkflow(member))) { + assert.equal(step.release, undefined, `${member} must leave release at its default`) + } + } + continue + } + + assert.ok(callCount > 1, `${file} no longer calls its reusable job more than once`) + const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member))) + const released = steps.filter((step) => step.release === "'true'") + assert.equal(released.length, 1, `${file} must free the run lease exactly once`) + for (const step of steps) { + if (step === released[0]) continue + assert.equal(step.release, "'false'", `${file} wave jobs must hold the lease`) + } + + const callerJobs = jobs(readWorkflow(file)) + const releaseJob = leaseStepsByJob(file).find((job) => + job.steps.some((step) => step.release === "'true'") + ) + assert.ok(releaseJob, `${file} must free the lease from its own job`) + const guard = jobIf(callerJobs.find((job) => job.id === releaseJob.id).text) + assert.match(guard, /always\(\)/, `${file}: the release job must run on failure and cancellation`) + + const holders = callerJobs + .filter( + (job) => + job.id !== releaseJob.id && + (waveFiles.some((member) => job.text.includes(`uses: ./.github/workflows/${member}`)) || + leaseStepsByJob(file).find((entry) => entry.id === job.id)?.steps.length > 0) + ) + .map((job) => job.id) + const needs = jobNeeds(callerJobs.find((job) => job.id === releaseJob.id).text) + for (const holder of holders) { + assert.ok(needs.includes(holder), `${file}: the release job must need ${holder}`) + } + } +}) + +test('workflows with two lease-holding jobs can never run them together', () => { + for (const file of leasedFiles) { + const entry = LEASED_WORKFLOWS[file] + if (entry.reentrant) continue + const holding = leaseStepsByJob(file).filter((job) => job.steps.length > 0) + if (holding.length <= 1) continue + assert.ok(entry.exclusiveBy, `${file} has ${holding.length} lease-holding jobs and no exclusivity guard`) + const guards = holding.map((job) => jobIf(jobs(readWorkflow(file)).find((j) => j.id === job.id).text)) + assert.equal( + guards.filter((guard) => guard.includes(entry.exclusiveBy)).length, + 1, + `${file}: exactly one job may run when ${entry.exclusiveBy}` + ) + assert.equal( + guards.filter((guard) => guard.includes(entry.exclusiveBy.replace('==', '!='))).length, + guards.length - 1, + `${file}: every other lease-holding job must be excluded when ${entry.exclusiveBy}` + ) + } +}) + +test('census: no workflow rolls out against the shared instance outside the lease', () => { + const minters = revisionMintingScripts() + assert.ok(minters.size > 0, 'the revision-minting script scan found nothing and is vacuous') + const flagged = new Map() + for (const file of workflowFiles()) { + const reason = mutatesSharedInstance(readWorkflow(file), minters) + if (reason) flagged.set(file, reason) + } + assert.ok(flagged.size > 0, 'the rollout census found no candidates and is vacuous') + + for (const [file, reason] of flagged) { + for (const entrypoint of entrypointsFor(file)) { + assert.ok( + entrypoint in LEASED_WORKFLOWS || entrypoint in NOT_A_CLOUD_SQL_CANDIDATE, + `${entrypoint} reaches ${file} (${reason}) but is neither leased nor recorded as a non-candidate` + ) + } + } + + for (const file of workflowFiles()) { + const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group) + if (!groups.some((group) => LOCK_GROUPS.has(group))) continue + assert.ok( + file in LEASED_WORKFLOWS || file in NOT_A_CLOUD_SQL_CANDIDATE, + `${file} sits in a Cloud SQL lock group but is neither leased nor recorded as a non-candidate` + ) + } + + for (const [file, reason] of Object.entries(NOT_A_CLOUD_SQL_CANDIDATE)) { + assert.ok(typeof reason === 'string' && reason.length > 40, `${file} needs a real reason`) + const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group) + assert.ok( + flagged.has(file) || groups.some((group) => LOCK_GROUPS.has(group)), + `${file} is recorded as a non-candidate but nothing would have flagged it` + ) + } + + for (const file of leasedFiles) { + assert.doesNotThrow(() => readWorkflow(file), `${file} is leased but does not exist`) + assert.ok(!(file in NOT_A_CLOUD_SQL_CANDIDATE), `${file} cannot be both leased and a non-candidate`) + } +}) + +// The API and auth deploy scripts share this contract but stay in the private repository. +const serviceCapScripts = ['dev/scripts/deploy-relay-blue-green.mjs'] + +test('budgets tagged Cloud Run candidates outside the service-wide instance cap', () => { + for (const file of serviceCapScripts) { + const script = readFileSync(new URL(`../../${file}`, import.meta.url), 'utf8') + assert.match(script, /'--no-traffic'/, file) + assert.match(script, /'--max'/, file) + } + const budget = readFileSync( + new URL('../../dev/scripts/relay-cloud-sql-connection-budget.mjs', import.meta.url), + 'utf8' + ) + assert.match(budget, /directly addressable tagged revisions outside service-level caps/) + assert.match( + budget, + /apiCandidate: retainedDirectorRollback \+ inputs\.apiInstances \* inputs\.apiPoolMax/ + ) + const director = readWorkflow(relayWorkflowFile('deploy-relay-production-director.yml')) + const capacity = readWorkflow(relayWorkflowFile('deploy-relay-production-capacity-job.yml')) + const asia = readWorkflow(relayWorkflowFile('operate-relay-asia-admission.yml')) + assert.match(director, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/) + assert.match(capacity, /--max-instances 5/) + assert.match(asia, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/) +}) diff --git a/cloud/dev/scripts/read-relay-production-capacity-identity.mjs b/cloud/dev/scripts/read-relay-production-capacity-identity.mjs new file mode 100644 index 00000000000..764b2732cfb --- /dev/null +++ b/cloud/dev/scripts/read-relay-production-capacity-identity.mjs @@ -0,0 +1,31 @@ +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +const CAPACITY_IDENTITY_NAME = 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT' + +export function readProductionCapacityIdentity(revision) { + const env = revision?.spec?.containers?.[0]?.env + if (!Array.isArray(env)) throw new Error('director revision environment is missing') + const matches = env.filter((entry) => entry?.name === CAPACITY_IDENTITY_NAME) + if (matches.length === 0) return null + if (matches.length !== 1) throw new Error('duplicate capacity identity') + const value = matches[0]?.value + if (typeof value !== 'string' || value.length === 0) { + throw new Error('capacity identity is not a literal string') + } + return value +} + +export function main() { + const revision = JSON.parse(readFileSync(0, 'utf8')) + process.stdout.write(`${JSON.stringify(readProductionCapacityIdentity(revision))}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + main() + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } +} diff --git a/cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs b/cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs new file mode 100644 index 00000000000..c75ddcbaf77 --- /dev/null +++ b/cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { readProductionCapacityIdentity } from './read-relay-production-capacity-identity.mjs' + +function revision(env) { + return { spec: { containers: [{ env }] } } +} + +test('reads absent, exact, and foreign literal capacity identities', () => { + assert.equal(readProductionCapacityIdentity(revision([])), null) + assert.equal( + readProductionCapacityIdentity(revision([ + { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'capacity@example.test' } + ])), + 'capacity@example.test' + ) + assert.equal( + readProductionCapacityIdentity(revision([ + { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'foreign@example.test' } + ])), + 'foreign@example.test' + ) +}) + +test('rejects malformed or duplicate capacity identity entries', () => { + for (const entry of [ + { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: null }, + { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: '' }, + { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', valueSource: { secretKeyRef: {} } } + ]) { + assert.throws( + () => readProductionCapacityIdentity(revision([entry])), + /not a literal string/ + ) + } + assert.throws( + () => readProductionCapacityIdentity(revision([ + { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'one@example.test' }, + { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'two@example.test' } + ])), + /duplicate capacity identity/ + ) + assert.throws(() => readProductionCapacityIdentity({}), /environment is missing/) +}) diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs new file mode 100644 index 00000000000..80d40277e5a --- /dev/null +++ b/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs @@ -0,0 +1,106 @@ +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const SECRET = 'orca-cloud-relay-regional-placement-enabled' + +function validate(input) { + for (const key of ['project', 'region', 'service', 'bootstrap_version']) { + if (typeof input?.[key] !== 'string' || !input[key] || /[\r\n]/.test(input[key])) { + throw new Error(`invalid ${key}`) + } + } + if (!/^[a-z][a-z0-9-]{0,62}$/.test(input.service)) throw new Error('invalid service') + if (!/^[1-9][0-9]*$/.test(input.bootstrap_version)) { + throw new Error('invalid bootstrap_version') + } +} + +export function classifyRelayServiceDescribeFailure(args, stderr) { + const serviceDescribe = args[0] === 'run' && args[1] === 'services' && args[2] === 'describe' + if (serviceDescribe && (stderr.includes('NOT_FOUND') || /Cannot find service \[[^\]\r\n]+\]/.test(stderr))) { + return 'NOT_FOUND' + } + return 'GCLOUD_FAILED' +} + +function defaultRun(args) { + const result = spawnSync('gcloud', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + maxBuffer: 10 * 1024 * 1024 + }) + if (result.status !== 0) { + const error = new Error('gcloud read failed') + error.code = classifyRelayServiceDescribeFailure(args, result.stderr) + throw error + } + return JSON.parse(result.stdout) +} + +function gcloudArguments(kind, input, revision) { + return [ + 'run', kind, 'describe', revision ?? input.service, + '--project', input.project, + '--region', input.region, + '--format=json' + ] +} + +export function readRelayServingRegionalPlacementVersion(input, dependencies = {}) { + validate(input) + const run = dependencies.run ?? defaultRun + let service + try { + service = run(gcloudArguments('services', input)) + } catch (error) { + if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version } + throw error + } + const serving = (service.status?.traffic ?? []).filter( + (entry) => Number(entry.percent ?? 0) > 0 + ) + if ( + serving.length !== 1 || + Number(serving[0].percent) !== 100 || + typeof serving[0].revisionName !== 'string' + ) { + throw new Error('Relay director must have exactly one revision serving 100% traffic') + } + const revision = run(gcloudArguments('revisions', input, serving[0].revisionName)) + const references = (revision.spec?.containers ?? []).flatMap((container) => + (container.env ?? []).filter( + (environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' + ) + ) + if (references.length === 0) return { version: input.bootstrap_version } + const reference = normalizeSecretReference(references[0]) + if ( + references.length !== 1 || + reference?.secret !== SECRET || + !/^[1-9][0-9]*$/.test(reference?.version ?? '') + ) { + throw new Error('serving regional placement secret reference is invalid') + } + return { version: reference.version } +} + +// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but +// `gcloud run revisions describe --format=json` emits the Knative v1 shape +// `valueFrom.secretKeyRef.{name,key}`, where `name` may be a full resource path. +// A `key` of "latest" is deliberately left invalid: the director's serving +// version must be a pinned integer for this data source to mean anything. +export function normalizeSecretReference(environment) { + const v2 = environment?.valueSource?.secretKeyRef + if (v2) return { secret: v2.secret, version: v2.version } + const v1 = environment?.valueFrom?.secretKeyRef + if (!v1) return undefined + const secret = typeof v1.name === 'string' ? v1.name.replace(/^projects\/[^/]+\/secrets\//, '') : undefined + return { secret, version: v1.key } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const chunks = [] + for await (const chunk of process.stdin) chunks.push(chunk) + const input = JSON.parse(Buffer.concat(chunks).toString('utf8')) + process.stdout.write(`${JSON.stringify(readRelayServingRegionalPlacementVersion(input))}\n`) +} diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs new file mode 100644 index 00000000000..8043fc23e94 --- /dev/null +++ b/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs @@ -0,0 +1,138 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + classifyRelayServiceDescribeFailure, + readRelayServingRegionalPlacementVersion +} from './read-relay-serving-regional-placement-version.mjs' + +const input = { + project: 'onorca-cloud', + region: 'us-central1', + service: 'orca-cloud-relay', + bootstrap_version: '7' +} + +function revision(version = '11') { + return { + spec: { + containers: [{ + env: [{ + name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED', + valueSource: { + secretKeyRef: { + secret: 'orca-cloud-relay-regional-placement-enabled', + version + } + } + }] + }] + } + } +} + +// Why: `gcloud run revisions describe --format=json` emits the Knative v1 shape, where the +// secret lives in `name` and the version in `key`, and `name` may be the full resource path. +function v1Revision(name, key) { + return { + spec: { + containers: [{ + env: [{ + name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED', + valueFrom: { secretKeyRef: { name, key } } + }] + }] + } + } +} + +function serving() { + return { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } +} + +test('reads the exact version from the sole traffic-serving revision', () => { + const calls = [] + const result = readRelayServingRegionalPlacementVersion(input, { + run: (args) => { + calls.push(args) + return calls.length === 1 + ? { + status: { + traffic: [ + { revisionName: 'relay-failed-latest', tag: 'candidate' }, + { revisionName: 'relay-serving', percent: 100 } + ] + } + } + : revision() + } + }) + + assert.deepEqual(result, { version: '11' }) + assert.equal(calls[1][3], 'relay-serving') +}) + +test('reads the gcloud v1 secret reference shape by bare id and by full resource path', () => { + for (const name of [ + 'orca-cloud-relay-regional-placement-enabled', + 'projects/120364513935/secrets/orca-cloud-relay-regional-placement-enabled' + ]) { + assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1') + }), { version: '1' }) + } +}) + +test('rejects a v1 reference that names another secret or a floating version', () => { + assert.throws(() => readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' + ? serving() + : v1Revision('projects/120364513935/secrets/some-other-secret', '1') + }), /secret reference is invalid/) + assert.throws(() => readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' + ? serving() + : v1Revision('orca-cloud-relay-regional-placement-enabled', 'latest') + }), /secret reference is invalid/) +}) + +test('falls back only when the service or setting is absent', () => { + const notFound = new Error('not found') + notFound.code = 'NOT_FOUND' + assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { + run: () => { throw notFound } + }), { version: '7' }) + assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' + ? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } + : { spec: { containers: [{ env: [] }] } } + }), { version: '7' }) +}) + +test('classifies real absent-service stderr without weakening revision failures', () => { + const serviceArgs = ['run', 'services', 'describe', 'missing-service'] + const stderr = 'ERROR: (gcloud.run.services.describe) Cannot find service [missing-service]' + assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, stderr), 'NOT_FOUND') + assert.equal( + classifyRelayServiceDescribeFailure(['run', 'revisions', 'describe', 'missing-revision'], stderr), + 'GCLOUD_FAILED' + ) + assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, 'PERMISSION_DENIED'), 'GCLOUD_FAILED') +}) + +test('rejects ambiguous traffic, malformed references, and read failures', () => { + assert.throws(() => readRelayServingRegionalPlacementVersion(input, { + run: () => ({ + status: { traffic: [{ revisionName: 'a', percent: 50 }, { revisionName: 'b', percent: 50 }] } + }) + }), /exactly one revision/) + assert.throws(() => readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' + ? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } + : revision('latest') + }), /secret reference is invalid/) + const denied = new Error('denied') + denied.code = 'GCLOUD_FAILED' + assert.throws(() => readRelayServingRegionalPlacementVersion(input, { + run: () => { throw denied } + }), denied) +}) diff --git a/cloud/dev/scripts/relay-admission-selector.mjs b/cloud/dev/scripts/relay-admission-selector.mjs new file mode 100644 index 00000000000..f41528c7e7f --- /dev/null +++ b/cloud/dev/scripts/relay-admission-selector.mjs @@ -0,0 +1,277 @@ +import { createHash } from 'node:crypto' + +const STATES = ['existing-only', 'migration-only', 'general'] + +function normalizeMembership(input) { + const membership = { + existingOnly: [...input.existingOnly].sort(), + migrationOnly: [...input.migrationOnly].sort(), + general: [...input.general].sort() + } + const all = [...membership.existingOnly, ...membership.migrationOnly, ...membership.general] + if (new Set(all).size !== all.length) throw new Error('selector membership contains duplicates') + return membership +} + +function encodedMembership(membership) { + return JSON.stringify(normalizeMembership(membership)) +} + +function membershipSha256(membership) { + return createHash('sha256').update(encodedMembership(membership)).digest('hex') +} + +function normalizeMigrationCells(input) { + const cells = [...input] + .map((cell) => ({ + cellId: cell.cellId, + cellUrl: cell.cellUrl, + capacityRequests: cell.capacityRequests, + ...(cell.region ? { region: cell.region } : {}), + connectionHardCap: cell.connectionHardCap, + connectionUnobservedBound: cell.connectionUnobservedBound + })) + .sort((left, right) => left.cellId.localeCompare(right.cellId)) + if ( + cells.length === 0 || + new Set(cells.map(({ cellId }) => cellId)).size !== cells.length || + new Set(cells.map(({ cellUrl }) => cellUrl)).size !== cells.length + ) { + throw new Error('migration cell registration must contain distinct cells') + } + return cells +} + +function membershipWithMigrationCells(membership, cells) { + const known = new Set([ + ...membership.existingOnly, + ...membership.migrationOnly, + ...membership.general + ]) + if (cells.some(({ cellId }) => known.has(cellId))) { + throw new Error('migration cell registration contains an existing selector cell') + } + return normalizeMembership({ + existingOnly: membership.existingOnly, + migrationOnly: [...membership.migrationOnly, ...cells.map(({ cellId }) => cellId)], + general: membership.general + }) +} + +function assertSelector(value) { + if ( + !value || + !Number.isSafeInteger(value.generation) || + value.generation < 0 || + !value.membership + ) { + throw new Error('director returned an invalid admission selector') + } + return { + generation: value.generation, + attemptId: value.attemptId ?? null, + membership: normalizeMembership(value.membership) + } +} + +export function selectorAttemptId(expectedGeneration, membership) { + const digest = createHash('sha256') + .update(`${expectedGeneration}:${encodedMembership(membership)}`) + .digest('hex') + .slice(0, 24) + return `selector_${expectedGeneration}_${digest}` +} + +export function membershipWithStates(selector, states) { + const byCell = new Map() + for (const [state, key] of [ + ['existing-only', 'existingOnly'], + ['migration-only', 'migrationOnly'], + ['general', 'general'] + ]) { + for (const cellId of selector.membership[key]) byCell.set(cellId, state) + } + for (const [cellId, state] of Object.entries(states)) { + if (!byCell.has(cellId)) throw new Error(`selector does not contain ${cellId}`) + if (!STATES.includes(state)) throw new Error(`invalid admission state for ${cellId}`) + if (byCell.get(cellId) === 'existing-only' && state !== 'existing-only') { + throw new Error(`selector cannot re-enable existing-only cell ${cellId}`) + } + byCell.set(cellId, state) + } + return normalizeMembership({ + existingOnly: [...byCell].filter(([, state]) => state === 'existing-only').map(([id]) => id), + migrationOnly: [...byCell].filter(([, state]) => state === 'migration-only').map(([id]) => id), + general: [...byCell].filter(([, state]) => state === 'general').map(([id]) => id) + }) +} + +export async function inspectAdmissionSelector(post, attemptId) { + const result = await post('/v1/admin/admission-selector/status', { + v: 1, + ...(attemptId ? { attemptId } : {}) + }) + return { + selector: assertSelector(result.selector), + intent: result.intent + ? { + ...result.intent, + previousMembership: result.intent.previousMembership + ? normalizeMembership(result.intent.previousMembership) + : undefined, + membership: normalizeMembership(result.intent.membership) + } + : null + } +} + +function exactSelector(actual, expected) { + return ( + actual.generation === expected.generation && + encodedMembership(actual.membership) === encodedMembership(expected.membership) + ) +} + +export async function applyExactAdmissionSelector(post, membership, options = {}) { + const before = await inspectAdmissionSelector(post) + const desired = normalizeMembership(membership) + if ( + options.expectedCurrentSelector && + !exactSelector(before.selector, options.expectedCurrentSelector) + ) { + throw new Error('admission selector changed before exact apply') + } + if (options.requireBoundary !== false && before.selector.generation < 1) { + throw new Error('admission selector boundary is not active') + } + if (encodedMembership(before.selector.membership) === encodedMembership(desired)) { + return { changed: false, selector: before.selector } + } + const attemptId = + options.attemptId ?? selectorAttemptId(before.selector.generation, desired) + const expected = { + generation: before.selector.generation + 1, + membership: desired + } + let result + try { + result = await post('/v1/admin/admission-selector/apply', { + v: 1, + attemptId, + expectedGeneration: before.selector.generation, + ...(before.selector.generation === 0 + ? { expectedMembershipSha256: membershipSha256(before.selector.membership) } + : {}), + membership: desired + }) + } catch (error) { + const inspected = await inspectAdmissionSelector(post, attemptId) + if ( + inspected.intent?.state === 'committed' && + exactSelector(inspected.selector, expected) + ) { + return { changed: true, selector: inspected.selector, recovered: true } + } + if ( + inspected.intent?.state === 'unchanged' && + exactSelector(inspected.selector, before.selector) + ) { + throw new Error('admission selector apply remained unchanged after an ambiguous response', { + cause: error + }) + } + throw new Error('admission selector apply diverged after an ambiguous response', { + cause: error + }) + } + const applied = assertSelector(result.selector) + if (!exactSelector(applied, expected)) { + throw new Error('admission selector apply returned unexpected membership') + } + const verified = await inspectAdmissionSelector(post, attemptId) + if ( + verified.intent?.state !== 'committed' || + !exactSelector(verified.selector, expected) + ) { + throw new Error('admission selector commit could not be verified') + } + return { changed: result.changed === true, selector: verified.selector } +} + +export async function addExactMigrationCells(post, input, options = {}) { + const cells = normalizeMigrationCells(input.cells) + const attemptId = input.attemptId + if (!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) { + throw new Error('migration cell registration requires an exact attempt ID') + } + const before = await inspectAdmissionSelector(post, attemptId) + let expectedGeneration + let expectedMembership + if (before.intent) { + expectedGeneration = before.intent.expectedGeneration + expectedMembership = normalizeMembership(before.intent.membership) + } else { + if (before.selector.generation < 1) { + throw new Error('admission selector boundary is not active') + } + if ( + options.expectedCurrentSelector && + !exactSelector(before.selector, options.expectedCurrentSelector) + ) { + throw new Error('admission selector changed before cell registration') + } + expectedGeneration = before.selector.generation + expectedMembership = membershipWithMigrationCells(before.selector.membership, cells) + } + const expected = { + generation: expectedGeneration + 1, + membership: expectedMembership + } + let result + try { + result = await post('/v1/admin/admission-selector/add-migration-cells', { + v: 1, + attemptId, + expectedGeneration, + cells + }) + } catch (error) { + const inspected = await inspectAdmissionSelector(post, attemptId) + if ( + !before.intent && + inspected.intent?.state === 'committed' && + exactSelector(inspected.selector, expected) + ) { + return { changed: true, selector: inspected.selector, recovered: true } + } + throw new Error('migration cell registration did not commit exactly', { cause: error }) + } + const applied = assertSelector(result.selector) + if (!exactSelector(applied, expected)) { + throw new Error('migration cell registration returned unexpected membership') + } + const verified = await inspectAdmissionSelector(post, attemptId) + if (verified.intent?.state !== 'committed' || !exactSelector(verified.selector, expected)) { + throw new Error('migration cell registration commit could not be verified') + } + return { changed: result.changed === true, selector: verified.selector } +} + +export async function transitionAdmissionSelector(post, states, options = {}) { + const current = await inspectAdmissionSelector(post) + if (current.selector.generation < 1) { + throw new Error('admission selector boundary is not active') + } + return await applyExactAdmissionSelector( + post, + membershipWithStates(current.selector, states), + options + ) +} + +export function selectorCellState(selector, cellId) { + if (selector.membership.existingOnly.includes(cellId)) return 'existing-only' + if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only' + if (selector.membership.general.includes(cellId)) return 'general' + throw new Error(`selector does not contain ${cellId}`) +} diff --git a/cloud/dev/scripts/relay-admission-selector.test.mjs b/cloud/dev/scripts/relay-admission-selector.test.mjs new file mode 100644 index 00000000000..b45df5ac34b --- /dev/null +++ b/cloud/dev/scripts/relay-admission-selector.test.mjs @@ -0,0 +1,232 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { test } from 'node:test' +import { + addExactMigrationCells, + applyExactAdmissionSelector, + membershipWithStates, + selectorAttemptId, + transitionAdmissionSelector +} from './relay-admission-selector.mjs' + +const initialMembership = { + existingOnly: ['legacy'], + migrationOnly: ['target'], + general: ['general'] +} + +function selectorHarness({ ambiguous = null, generation = 1 } = {}) { + let selector = { generation, attemptId: 'initial', membership: initialMembership } + const intents = new Map() + const requests = [] + let applies = 0 + const post = async (path, body) => { + if (path.endsWith('/status')) { + return { + selector, + intent: body.attemptId ? intents.get(body.attemptId) ?? null : null + } + } + applies++ + requests.push(body) + const before = selector + const committed = { + generation: body.expectedGeneration + 1, + attemptId: body.attemptId, + membership: body.membership + } + intents.set(body.attemptId, { + attemptId: body.attemptId, + expectedGeneration: body.expectedGeneration, + intendedGeneration: committed.generation, + previousMembership: before.membership, + membership: body.membership, + state: ambiguous === 'unchanged' ? 'unchanged' : 'committed' + }) + if (ambiguous !== 'unchanged') selector = committed + if (ambiguous) throw new Error('lost selector response') + return { changed: true, selector } + } + return { post, selector: () => selector, applies: () => applies, requests } +} + +test('derives deterministic attempts and applies exact selector transitions', async () => { + const harness = selectorHarness() + const desired = membershipWithStates(harness.selector(), { target: 'general' }) + assert.equal( + selectorAttemptId(1, desired), + selectorAttemptId(1, { + existingOnly: ['legacy'], + migrationOnly: [], + general: ['target', 'general'] + }) + ) + const result = await transitionAdmissionSelector(harness.post, { target: 'general' }) + assert.equal(result.selector.generation, 2) + assert.deepEqual(result.selector.membership.general, ['general', 'target']) +}) + +test('accepts only an exact committed result after an ambiguous response', async () => { + const harness = selectorHarness({ ambiguous: 'committed' }) + const result = await applyExactAdmissionSelector(harness.post, { + existingOnly: ['legacy', 'target'], + migrationOnly: [], + general: ['general'] + }) + assert.equal(result.recovered, true) + assert.equal(harness.applies(), 1) + assert.equal(result.selector.generation, 2) +}) + +test('binds a generation-zero cutover to the inspected membership', async () => { + const harness = selectorHarness({ generation: 0 }) + await applyExactAdmissionSelector( + harness.post, + { + existingOnly: ['legacy', 'target'], + migrationOnly: [], + general: ['general'] + }, + { requireBoundary: false } + ) + assert.equal( + harness.requests[0].expectedMembershipSha256, + createHash('sha256').update(JSON.stringify(initialMembership)).digest('hex') + ) +}) + +test('stops on an unchanged ambiguous result without replaying', async () => { + const harness = selectorHarness({ ambiguous: 'unchanged' }) + await assert.rejects( + applyExactAdmissionSelector(harness.post, { + existingOnly: ['legacy', 'target'], + migrationOnly: [], + general: ['general'] + }), + /remained unchanged/ + ) + assert.equal(harness.applies(), 1) + assert.equal(harness.selector().generation, 1) +}) + +test('never restores an existing-only cell', () => { + assert.throws( + () => membershipWithStates({ membership: initialMembership }, { legacy: 'general' }), + /cannot re-enable/ + ) +}) + +test('refuses an exact apply after the inspected selector changes', async () => { + const harness = selectorHarness() + await assert.rejects( + applyExactAdmissionSelector( + harness.post, + { + existingOnly: ['legacy', 'target'], + migrationOnly: [], + general: ['general'] + }, + { + expectedCurrentSelector: { + generation: 0, + membership: { + existingOnly: ['target'], + migrationOnly: [], + general: ['general', 'legacy'] + } + } + } + ), + /changed before exact apply/ + ) + assert.equal(harness.applies(), 0) +}) + +test('adds exact migration cells and recovers a committed response loss', async () => { + let selector = { generation: 1, attemptId: 'initial', membership: initialMembership } + const intents = new Map() + let additions = 0 + const post = async (path, body) => { + if (path.endsWith('/status')) { + return { + selector, + intent: body.attemptId ? intents.get(body.attemptId) ?? null : null + } + } + additions++ + selector = { + generation: body.expectedGeneration + 1, + attemptId: body.attemptId, + membership: { + ...selector.membership, + migrationOnly: [ + ...selector.membership.migrationOnly, + ...body.cells.map(({ cellId }) => cellId) + ].sort() + } + } + intents.set(body.attemptId, { + attemptId: body.attemptId, + expectedGeneration: body.expectedGeneration, + intendedGeneration: selector.generation, + membership: selector.membership, + state: 'committed' + }) + throw new Error('lost cell registration response') + } + const result = await addExactMigrationCells(post, { + attemptId: 'add_cells_exact', + cells: [ + { + cellId: 'target-2', + cellUrl: 'https://target-2.example.com', + capacityRequests: 4_000, + connectionHardCap: 600, + connectionUnobservedBound: 60 + } + ] + }) + assert.equal(result.recovered, true) + assert.equal(additions, 1) + assert.deepEqual(result.selector.membership.migrationOnly, ['target', 'target-2']) +}) + +test('does not recover an attempt owned by another selector operation', async () => { + const selector = { + generation: 2, + attemptId: 'selector_collision', + membership: initialMembership + } + const post = async (path, body) => { + if (path.endsWith('/status')) { + return { + selector, + intent: body.attemptId + ? { + attemptId: body.attemptId, + expectedGeneration: 1, + intendedGeneration: 2, + membership: initialMembership, + state: 'committed' + } + : null + } + } + throw new Error('admission_selector_attempt_mismatch') + } + await assert.rejects( + addExactMigrationCells(post, { + attemptId: 'selector_collision', + cells: [ + { + cellId: 'target-2', + cellUrl: 'https://target-2.example.com', + capacityRequests: 4_000, + connectionHardCap: 600, + connectionUnobservedBound: 60 + } + ] + }), + /did not commit exactly/ + ) +}) diff --git a/cloud/dev/scripts/relay-asia-admission-workflow.test.mjs b/cloud/dev/scripts/relay-asia-admission-workflow.test.mjs new file mode 100644 index 00000000000..21d712601d0 --- /dev/null +++ b/cloud/dev/scripts/relay-asia-admission-workflow.test.mjs @@ -0,0 +1,323 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { relayWorkflowUrl } from './relay-repository.mjs' + +const workflow = readFileSync( + relayWorkflowUrl('operate-relay-asia-admission.yml'), + 'utf8' +) +const iam = readFileSync(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url), 'utf8') +const stagingProof = readFileSync( + relayWorkflowUrl('prove-relay-asia-staging.yml'), + 'utf8' +) +const directorWorkflow = readFileSync( + relayWorkflowUrl('deploy-relay-production-director.yml'), + 'utf8' +) +const terraformReadme = readFileSync( + new URL('../../infra/terraform/README.md', import.meta.url), + 'utf8' +) +const proofIam = readFileSync( + new URL('../../infra/terraform/relay-asia-proof-iam.tf', import.meta.url), + 'utf8' +) +const relayTerraform = readFileSync( + new URL('../../infra/terraform/relay.tf', import.meta.url), + 'utf8' +) +const rolloutEvidence = readFileSync( + new URL('./relay-asia-rollout-evidence.mjs', import.meta.url), + 'utf8' +) +const admissionBudgets = readFileSync( + new URL('../../packages/relay-contract/src/admission-budgets.ts', import.meta.url), + 'utf8' +) + +test('offers the exact audited admission modes under the shared deployment lock', () => { + for (const mode of [ + 'inspect', 'initialize', 'verify', 'register', 'configure', 'promote', 'rollback' + ]) { + assert.match(workflow, new RegExp(`\\b${mode}\\b`)) + } + assert.match(workflow, /production-cloud-sql-rollout/) + assert.match(workflow, /relay-staging-mutation/) + assert.match(workflow, /selector-generation/) + assert.match(workflow, /selector-attempt-id/) +}) + +test('requires exact confirmations and uses the existing admin identity', () => { + assert.match(workflow, /INITIALIZE_ADMISSION_SELECTOR/) + assert.match(workflow, /REGISTER_ASIA_MIGRATION_ONLY/) + assert.match(workflow, /PROMOTE_ASIA_GENERAL/) + assert.match(workflow, /ROLLBACK_ASIA_MIGRATION_ONLY/) + assert.match(workflow, /CONFIGURE_ASIA_DIRECTOR/) + assert.match(workflow, /GCP_RELAY_DEPLOY_SERVICE_ACCOUNT/) + assert.match(workflow, /id_token_audience: \$\{\{ env\.DIRECTOR_ORIGIN \}\}\/v1\/admin\/drain/) + assert.match(iam, /"operate-relay-asia-admission\.yml"/) +}) + +test('discovers generation read-only and explicitly initializes only generation zero', () => { + assert.match(workflow, /leave empty only for inspect/) + assert.match(workflow, /test -z "\$\{EXPECTED_SELECTOR_GENERATION\}"/) + assert.match(workflow, /test "\$\{EXPECTED_SELECTOR_GENERATION\}" = 0/) + assert.match(workflow, /\^\(0\|\[1-9\]\[0-9\]\*\)\$/) + assert.match(workflow, /selector-membership-sha256/) + assert.match(workflow, /\^\[a-f0-9\]\{64\}\$/) + assert.match(workflow, /director-image-digest/) + assert.match(workflow, /\.spec\.containers\[0\]\.image == \$image/) +}) + +test('uploads one sanitized machine-readable admission result', () => { + assert.match(workflow, /sanitize-relay-asia-admission-result\.mjs/) + const upload = /- name: Upload sanitized admission result\n([\s\S]*?)(?=\n - name:)/ + .exec(workflow)?.[1] + assert.ok(upload) + assert.match( + upload, + /if: \$\{\{ inputs\.mode != 'configure' && steps\.admission-operation\.outcome == 'success' \}\}/ + ) + assert.match(upload, /uses: actions\/upload-artifact@v4/) + assert.match( + upload, + /relay-asia-admission-result-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/ + ) + assert.match(upload, /path: \$\{\{ runner\.temp \}\}\/relay-asia-admission-result\/result\.json/) + assert.match(upload, /if-no-files-found: error/) + assert.match(upload, /retention-days: 7/) + assert.ok( + workflow.indexOf('Upload sanitized admission result') > + workflow.indexOf('Upload immutable C27 canary evidence') + ) +}) + +test('binds selector operations and director configuration to reviewed implementations', () => { + assert.match(workflow, /operate-relay-asia-admission\.mjs/) + assert.match(workflow, /prepare-relay-asia-director-cells\.mjs/) + assert.match(workflow, /deploy-relay-blue-green\.mjs/) + assert.match(workflow, /--prune-revisions false/) + assert.doesNotMatch(workflow, /gcloud secrets versions add/) + assert.match(workflow, /orca-cloud-relay-regional-placement-enabled/) + assert.match(workflow, /\.valueSource\.secretKeyRef/) + assert.match(workflow, /jq -er --arg secret "\$\{REGIONAL_PLACEMENT_SECRET\}"/) + assert.doesNotMatch(workflow, /jq -e --arg secret "\$\{REGIONAL_PLACEMENT_SECRET\}"/) + assert.doesNotMatch(workflow, /--regional-placement-enabled/) + assert.doesNotMatch(workflow, /"\$\{\{ inputs\./) + assert.doesNotMatch(workflow, /dns/i) +}) + +test('requires immutable staged evidence and a timed C27 canary before expansion', () => { + assert.match(workflow, /actions: read/) + assert.match(workflow, /actions\/download-artifact@v4/) + assert.match(workflow, /relay-asia-staging-\$\{EVIDENCE_RUN_ID\}-\$\{EVIDENCE_RUN_ATTEMPT\}/) + assert.match(workflow, /evidence_kind=staging/) + assert.match(workflow, /load-relay-controls\.mjs/) + assert.match(workflow, /--controls 1/) + assert.match(workflow, /--splices 1/) + assert.match(workflow, /--splice-hold-seconds 60/) + assert.match(workflow, /--relay-asia-load-principals 1/) + assert.match(workflow, /--duration-seconds 300/) + assert.match(workflow, /--required-lease-horizons 2/) + assert.match(workflow, /pnpm\/action-setup@v4/) + assert.match(workflow, /Install exact C27 canary dependencies/) + assert.match(workflow, /pnpm install --frozen-lockfile/) + assert.match(workflow, /pnpm --filter @orca-cloud\/relay-contract build/) + assert.ok( + workflow.indexOf('Build the C27 canary Relay contract') < + workflow.indexOf('Run a real five-minute C27 control and splice canary') + ) + assert.match(workflow, /--load-report "\$\{RUNNER_TEMP\}\/relay-asia-c27-load\.json"/) + assert.match(workflow, /states\["production-gce-c28"\].*= migration-only/) + assert.match(workflow, /states\["production-gce-c29"\].*= migration-only/) + assert.match(workflow, /relay-asia-c27-canary-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/) + assert.match(workflow, /id: c27-evidence-upload/) + assert.match(workflow, /Return an unproven C27 canary to migration-only/) + assert.match(workflow, /steps\.c27-evidence-upload\.outcome != 'success'/) + assert.match(workflow, /--mode recover-promotion[\s\S]*?--attempt-id "\$\{SELECTOR_ATTEMPT_ID\}"/) + assert.match(workflow, /--attempt-id "\$\{SELECTOR_ATTEMPT_ID\}-rollback"/) + assert.match(workflow, /evidence_kind=c27/) + assert.match(workflow, /orca_relay_runtime_metrics/) + assert.match(workflow, /relay-asia-rollout-evidence\.mjs create-c27/) + assert.match(workflow, /retention-days: 7/) + assert.match(workflow, /Require the exact director image before promotion/) + assert.match(workflow, /DIRECTOR_ORIGIN.*\/v1\/admin\/runtime-status/) + assert.match(workflow, /\.imageDigest.*IMAGE_DIGEST/) + const provenance = /- name: Verify evidence provenance and rollout binding before authentication\n([\s\S]*?)(?=\n - id: auth)/ + .exec(workflow)?.[1] + assert.ok(provenance) + assert.match(provenance, /\.head_sha \| select\(type == "string" and test\("\^\[a-f0-9\]\{40\}\$"\)\)/) + assert.match(provenance, /--commit-sha "\$\{evidence_commit_sha\}"/) + assert.doesNotMatch(provenance, /--commit-sha "\$\{GITHUB_SHA\}"/) +}) + +test('creates staging evidence only after the bounded launch-path load and rollback', () => { + assert.match(stagingProof, /runs-on: \[self-hosted, linux, x64, relay-asia-east2-load\]/) + assert.doesNotMatch(stagingProof, /group: relay-asia-east2-load/) + assert.match(stagingProof, /pnpm\/action-setup@v4/) + assert.match(stagingProof, /pnpm install --frozen-lockfile/) + assert.match(stagingProof, /pnpm --filter @orca-cloud\/relay-contract build/) + assert.match(stagingProof, /run_phase launch 5 5/) + assert.doesNotMatch(stagingProof, /run_phase control|run_phase mixed/) + assert.match(stagingProof, /--aggregate-controls "\$\(\(controls \* 4\)\)"/) + assert.match(stagingProof, /--aggregate-splices "\$\(\(splices \* 4\)\)"/) + assert.match(stagingProof, /--required-lease-horizons 2/) + assert.match(stagingProof, /--splice-ramp-seconds 120/) + assert.match(stagingProof, /--max-generator-rss-growth-mib 512/) + assert.match(stagingProof, /--relay-asia-load-principals 32/) + assert.match(stagingProof, /ulimit -n/) + assert.match(stagingProof, /--region-behavior-probes 1/) + assert.match(stagingProof, /--capacity-cell-origin https:\/\/c4\.relay-staging\.onorca\.dev/) + assert.match(stagingProof, /--rebind-probes 2/) + assert.match(stagingProof, /--skip-rebind-overflow-check/) + assert.doesNotMatch(stagingProof, /--request-unit-invites|--regional-fallback-probes/) + assert.match(stagingProof, /--aggregate-reader-splices.*echo 5/) + assert.match(stagingProof, /--aggregate-reader-bytes.*echo 12582912/) + assert.match(stagingProof, /--phase-barrier-dir "\$\{proof_dir\}\/\$\{phase\}-barrier"/) + assert.match(stagingProof, /--duration-seconds 210/) + assert.match(stagingProof, /trap stop_shards EXIT/) + assert.match(stagingProof, /if ! wait "\$\{pid\}"; then failed=1; break; fi/) + assert.match(stagingProof, /connectionFailuresByReason/) + assert.match(stagingProof, /--launch-report "\$\{proof_dir\}\/launch\.json"/) + assert.match(stagingProof, /id-token: write/) + assert.match(stagingProof, /STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(stagingProof, /STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT/) + assert.doesNotMatch(stagingProof, /STAGING_GCP_DEPLOY_SERVICE_ACCOUNT/) + assert.doesNotMatch(stagingProof, /STAGING_RELAY_LOAD_ACCESS_TOKEN/) + assert.doesNotMatch(stagingProof, /secrets versions access|signing-key-file/) + assert.match(stagingProof, /relay-asia-rollout-evidence\.mjs create-staging/) + assert.match(stagingProof, /Require the exact staging director image before promotion/) + assert.match(stagingProof, /DIRECTOR_ORIGIN.*\/v1\/admin\/runtime-status/) + assert.match(stagingProof, /\.imageDigest.*IMAGE_DIGEST/) + assert.match(stagingProof, /Return staging C4 to migration-only/) + assert.match(stagingProof, /steps\.promote\.outcome != 'skipped'/) + assert.match(stagingProof, /--mode recover-promotion[\s\S]*?--attempt-id "\$\{PROMOTE_ATTEMPT_ID\}"/) + assert.match(stagingProof, /--mode rollback[\s\S]*?--expected-generation "\$\{promoted_generation\}"/) + assert.match(stagingProof, /if: \$\{\{ success\(\) \}\}/) + assert.match( + stagingProof, + /recover:\n if: \$\{\{ always\(\) && github\.ref == 'refs\/heads\/main' \}\}/ + ) + assert.match(stagingProof, /needs: prove/) + assert.match(stagingProof, /Recover staging C4 with a fresh identity/) + assert.equal((stagingProof.match(/google-github-actions\/auth@v2/g) ?? []).length, 2) + assert.equal((stagingProof.match(/--mode recover-promotion/g) ?? []).length, 2) + assert.equal((stagingProof.match(/--mode rollback/g) ?? []).length, 2) +}) + +test('keeps the private runner below its 64-port Cloud NAT allocation', () => { + const profile = /run_phase launch (\d+) (\d+)/.exec(stagingProof) + const controlsPerShard = Number(profile?.[1]) + const splicesPerShard = Number(profile?.[2]) + const rebindProbes = Number(/--rebind-probes (\d+)/.exec(stagingProof)?.[1]) + const runtimeStatusSockets = 1 + assert.ok( + controlsPerShard * 4 + splicesPerShard * 4 * 2 + rebindProbes + runtimeStatusSockets < 64 + ) +}) + +test('paces one-source staging upgrades below the Relay anti-abuse ceiling', () => { + const splicesPerShard = Number(/run_phase launch \d+ (\d+)/.exec(stagingProof)?.[1]) + const rebindProbes = Number(/--rebind-probes (\d+)/.exec(stagingProof)?.[1]) + const spliceRampMs = Number(/--splice-ramp-seconds (\d+)/.exec(stagingProof)?.[1]) * 1000 + const ceiling = Number( + /maxPreAuthAttemptsPerSourcePerMinute: (\d+)/.exec(admissionBudgets)?.[1] + ) + const totalSplices = splicesPerShard * 4 + const attempts = Array.from({ length: totalSplices }, (_, ordinal) => + Math.floor(ordinal * spliceRampMs / (totalSplices - 1)) + ).flatMap((startedAt) => [startedAt, startedAt]) + attempts.push(...Array.from({ length: 4 + rebindProbes }, () => 0)) + const busiestMinute = Math.max(...attempts.map((startedAt) => + attempts.filter((attempt) => attempt >= startedAt && attempt < startedAt + 60_000).length + )) + assert.ok(busiestMinute < ceiling) +}) + +test('reserves rollback time beyond the complete bounded staging proof envelope', () => { + const timeoutMinutes = Number(/timeout-minutes: (\d+)/.exec(stagingProof)?.[1]) + assert.equal(timeoutMinutes, 75) + const spliceRampSeconds = Number(/--splice-ramp-seconds (\d+)/.exec(stagingProof)?.[1]) + const launchSeconds = 180 + spliceRampSeconds + 210 + 60 + const setupEvidenceAndRollbackSeconds = 10 * 60 + const envelopeMinutes = Math.ceil((launchSeconds + setupEvidenceAndRollbackSeconds) / 60) + assert.ok(timeoutMinutes - envelopeMinutes >= 30) + assert.match(stagingProof, /--ramp-seconds 180/) + assert.match(stagingProof, /--duration-seconds 210/) +}) + +test('binds the staging proof to one least-privilege Google identity', () => { + assert.match( + proofIam, + /github_relay_asia_proof_workflow_file = "prove-relay-asia-staging\.yml"/ + ) + assert.match( + proofIam, + /assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_proof_workflow_file\}@refs\/heads\/main'/ + ) + assert.match(proofIam, /assertion\.environment == 'staging'/) + assert.match(proofIam, /assertion\.event_name == 'workflow_dispatch'/) + assert.match(proofIam, /roles\/logging\.viewer/) + assert.match(proofIam, /roles\/monitoring\.viewer/) + assert.match(rolloutEvidence, /readCloudSqlBackends/) + assert.match(rolloutEvidence, /cloudSql: await readCloudSqlBackends/) + assert.doesNotMatch(proofIam, /compute\.|cloudsql\.|secretmanager\.|roles\/editor|roles\/run\./) +}) + +test('keeps the production US-first switch in durable Secret Manager state', () => { + assert.match(directorWorkflow, /options: \[preserve, enable, disable\]/) + assert.match(directorWorkflow, /default: preserve/) + assert.match(directorWorkflow, /gcloud secrets versions add/) + assert.match(directorWorkflow, /preserve\) desired="\$\{current\}"/) + assert.match(directorWorkflow, /--regional-placement-secret-version "\$\{target_version\}"/) + assert.match(directorWorkflow, /test "\$\{CEILING\}" = "\$\{DIRECTOR_MAX_INSTANCES\}"/) + assert.match(directorWorkflow, /orca-cloud-relay-regional-placement-enabled/) + assert.match(directorWorkflow, /\.valueSource\.secretKeyRef \/\/ \.valueFrom\.secretKeyRef/) + assert.match(directorWorkflow, /\.version \/\/ \.key/) + assert.match(directorWorkflow, /\.secret \/\/ \.name/) + assert.match(workflow, /\.valueSource\.secretKeyRef \/\/ \.valueFrom\.secretKeyRef/) + assert.doesNotMatch(directorWorkflow, /--regional-placement-enabled/) + assert.doesNotMatch(workflow, /inputs\.regional-placement-enabled/) +}) + +test('prunes incompatible production revisions only when explicitly confirmed', () => { + assert.match( + directorWorkflow, + /prune-incompatible-revisions:[\s\S]*?default: false[\s\S]*?type: boolean/ + ) + assert.match(directorWorkflow, /PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS/) + assert.match( + directorWorkflow, + /test "\$\{REGIONAL_PLACEMENT_MODE\}" = preserve[\s\S]*?test "\$\{CONFIRMATION\}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS/ + ) + assert.match( + directorWorkflow, + /--prune-revisions "\$\{PRUNE_INCOMPATIBLE_REVISIONS\}"/ + ) +}) + +test('documents the exact regional-placement secret bootstrap before director rollout', () => { + for (const address of [ + 'google_secret_manager_secret.relay_regional_placement_enabled', + 'google_secret_manager_secret_version.relay_regional_placement_enabled', + 'google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor', + 'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor[0]', + 'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder[0]', + 'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer[0]' + ]) { + assert.match(terraformReadme, new RegExp(address.replaceAll(/[.[\]]/g, '\\$&'))) + } + assert.match(terraformReadme, /Before the first director deployment/) + assert.match(terraformReadme, /Pass the exact environment tfvars/) + // The Cloudflare records left with the apps root; a -var for a variable this root no longer + // declares is a hard error, so no relay procedure may still tell an operator to pass it. + assert.doesNotMatch(terraformReadme, /manage_artifact_dns/) + assert.match(terraformReadme, /exactly these six additions/) + assert.match(terraformReadme, /version metadata/) + assert.match( + relayTerraform, + /resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_viewer"[\s\S]*?role\s+= "roles\/secretmanager\.viewer"/ + ) +}) diff --git a/cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs b/cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs new file mode 100644 index 00000000000..489402b0399 --- /dev/null +++ b/cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs @@ -0,0 +1,33 @@ +import { spawnSync } from 'node:child_process' + +function accessToken() { + const result = spawnSync('gcloud', ['auth', 'print-access-token'], { + encoding: 'utf8', timeout: 30_000 + }) + const token = result.stdout.trim() + if (result.status !== 0 || token.length < 20) { + throw new Error('Google access token is unavailable') + } + return token +} + +export async function readCloudSqlBackends(environment, startedAt, endedAt) { + const production = environment === 'production' + if (!production && environment !== 'staging') throw new Error('Cloud SQL environment is invalid') + const project = production ? 'onorca-cloud' : 'onorca-cloud-staging' + const instance = production ? 'orca-cloud-auth-db' : 'orca-cloud-staging-auth-db' + const url = new URL(`https://monitoring.googleapis.com/v3/projects/${project}/timeSeries`) + url.searchParams.set('filter', `metric.type = "cloudsql.googleapis.com/database/postgresql/num_backends" AND resource.labels.database_id = "${project}:${instance}"`) + url.searchParams.set('interval.startTime', startedAt) + url.searchParams.set('interval.endTime', endedAt) + url.searchParams.set('aggregation.alignmentPeriod', '60s') + url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_MAX') + url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_MAX') + url.searchParams.set('view', 'FULL') + const response = await fetch(url, { + headers: { authorization: `Bearer ${accessToken()}` }, + redirect: 'error', signal: AbortSignal.timeout(30_000) + }) + if (!response.ok) throw new Error(`Cloud SQL metric query returned ${response.status}`) + return await response.json() +} diff --git a/cloud/dev/scripts/relay-asia-rollout-evidence.mjs b/cloud/dev/scripts/relay-asia-rollout-evidence.mjs new file mode 100644 index 00000000000..e833a0ae16b --- /dev/null +++ b/cloud/dev/scripts/relay-asia-rollout-evidence.mjs @@ -0,0 +1,544 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { readCloudSqlBackends } from './relay-asia-cloud-sql-metrics.mjs' +import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs' + +const REPOSITORY = RELAY_GITHUB_REPOSITORY +const ADMISSION_WORKFLOW = relayWorkflowPath('operate-relay-asia-admission.yml') +const STAGING_WORKFLOW = relayWorkflowPath('prove-relay-asia-staging.yml') +const STAGING_CELL = 'staging-gce-c4' +const C27 = 'production-gce-c27' +const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/ +const SHA_PATTERN = /^[a-f0-9]{40}$/ +const MAX_LOG_EDGE_GAP_MS = 120_000 +const MAX_LOG_SAMPLE_GAP_MS = 120_000 +const CLOUD_SQL_LIMIT = 320 +const C27_CANARY_MINIMUM_MS = 5 * 60_000 +const GENERATOR_CPU_PERCENT_LIMIT = 80 +const GENERATOR_EVENT_LOOP_P99_MS_LIMIT = 100 +const GENERATOR_RSS_GROWTH_MIB_LIMIT = 512 +const DATABASE_POOL_TRANSIENT_WAITERS_MAX = 4 +const DATABASE_POOL_TRANSIENT_WAIT_MS_MAX = 50 + +function object(value, label) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} is invalid`) + } + return value +} + +function positiveInteger(value, label) { + const number = Number(value) + if (!Number.isSafeInteger(number) || number < 1) throw new Error(`${label} is invalid`) + return number +} + +function instant(value, label) { + const date = new Date(value) + if (!Number.isFinite(date.valueOf())) throw new Error(`${label} is invalid`) + return date +} + +function exactCells(actual, expected) { + return Array.isArray(actual) && + JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort()) +} + +function source(input, environment, workflow) { + if (input.repository !== REPOSITORY) throw new Error('repository is invalid') + if (!SHA_PATTERN.test(input.commitSha)) throw new Error('commit SHA is invalid') + return { + repository: input.repository, + workflow, + environment, + runId: positiveInteger(input.runId, 'run ID'), + runAttempt: positiveInteger(input.runAttempt, 'run attempt'), + commitSha: input.commitSha + } +} + +function baseEvidence(input, environment, cells, workflow = ADMISSION_WORKFLOW) { + if (!DIGEST_PATTERN.test(input.imageDigest)) throw new Error('image digest is invalid') + return { + version: 1, + source: source(input, environment, workflow), + imageDigest: input.imageDigest, + topology: { + cellIds: cells, + selectorGeneration: positiveInteger(input.selectorGeneration, 'selector generation') + } + } +} + +export function buildStagingEvidence(input) { + const start = instant(input.startedAt, 'staging proof start') + const end = instant(input.endedAt, 'staging proof end') + const launch = loadReports(input.launchReport, 'launch load report') + const expectedLoad = { + controls: 20, splices: 20, slowReaders: 4, wedgedReaders: 1, + minimumSeconds: 210 + } + assertLoadReports(launch, expectedLoad) + const metrics = runtimeMetrics(input.logs, start, end, STAGING_CELL) + assertPassingRuntimeMetrics(metrics, 'staging proof', 1) + const minimumConcurrentSplices = expectedLoad.splices - expectedLoad.wedgedReaders + if ( + metrics.targetControlsMax < expectedLoad.controls || + metrics.targetSplicesMax < minimumConcurrentSplices + ) { + throw new Error('staging launch load did not reach C4 at the reviewed levels') + } + metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end) + if (metrics.cloudSqlBackendsMax >= CLOUD_SQL_LIMIT) { + throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`) + } + return { + ...baseEvidence(input, 'staging', [STAGING_CELL], STAGING_WORKFLOW), + kind: 'staging-asia-readiness', + window: { startedAt: start.toISOString(), endedAt: end.toISOString() }, + load: { launch: loadSummary(launch) }, + metrics + } +} + +function loadReports(value, label) { + if (!Array.isArray(value) || value.length < 2) throw new Error(`${label} must be sharded`) + return value.map((report) => object(report, label)) +} + +function total(reports, key) { + return reports.reduce((sum, report) => sum + number(report[key], key), 0) +} + +function loadSummary(reports) { + return { + shards: reports.length, + controls: total(reports, 'controls'), + peakActive: total(reports, 'peakActive'), + steadyMinimumActive: total(reports, 'steadyMinimumActive'), + peakActiveSplices: total(reports, 'peakActiveSplices'), + completedSplices: total(reports, 'completedSplices'), + slowReaderSplicesCompleted: total(reports, 'slowReaderSplicesCompleted'), + wedgedReaderSplicesClosed: total(reports, 'wedgedReaderSplicesClosed'), + regionalFallbacksProved: total(reports, 'regionalFallbacksProved'), + oldClientUsFirstProved: total(reports, 'oldClientUsFirstProved'), + stickyAssignmentProved: total(reports, 'stickyAssignmentProved'), + requestUnitInvitesOpened: total(reports, 'requestUnitInvitesOpened'), + requestUnitPrincipalCounts: reports.map((report) => report.requestUnitPrincipalCount), + requestUnitOverflowReasons: + reports.map((report) => report.requestUnitOverflowReason).filter(Boolean), + requestUnitCleanupProved: total(reports, 'requestUnitCleanupProved'), + phaseBarrierPassed: reports.every((report) => report.phaseBarrierPassed === true), + rebindProbesOpened: total(reports, 'rebindProbesOpened'), + rebindOverflowReasons: reports.map((report) => report.rebindOverflowReason).filter(Boolean), + readerQueueEvidence: reports.flatMap((report) => report.readerQueueEvidence), + readerQueuedBytesPeak: Math.max(...reports.map((report) => report.readerQueuedBytesPeak)), + generatorCpuPercentMax: Math.max(...reports.map((report) => report.generatorCpuPercent)), + generatorEventLoopP99MsMax: Math.max( + ...reports.map((report) => report.generatorEventLoopP99Ms) + ), + generatorRssGrowthMiBMax: Math.max(...reports.map((report) => report.generatorRssGrowthMiB)), + configuredSteadySeconds: Math.min(...reports.map((report) => report.configuredSteadySeconds)) + } +} + +function assertLoadReports(reports, expected) { + const shardCount = reports.length + if ( + reports.some((report, index) => + report.event !== 'relay_load_complete' || + report.shardCount !== shardCount || report.shardIndex !== index || + report.relayAsiaLoadPrincipalCount !== 32 || + number(report.configuredSteadySeconds, 'staging steady seconds') < expected.minimumSeconds || + number(report.configuredSpliceHoldSeconds, 'staging splice hold seconds') < + expected.minimumSeconds || + report.requiredLeaseHorizons !== 2 || report.phaseBarrierPassed !== true + ) || + total(reports, 'controls') !== expected.controls + ) { + throw new Error('staging load profile does not match') + } + if ( + total(reports, 'peakActive') !== expected.controls || + total(reports, 'steadyMinimumActive') !== expected.controls + ) { + throw new Error('staging load did not sustain the required controls') + } + for (const key of [ + 'rampConnectionFailures', 'steadyConnectionFailures', 'transitionConnectionFailures', + 'unexpectedCloses', 'protocolErrors', 'refreshErrors', 'socketErrors', 'failedSplices' + ]) { + if (total(reports, key) !== 0) throw new Error(`staging load ${key} must be zero`) + } + const peakActiveSplices = total(reports, 'peakActiveSplices') + if ( + total(reports, 'configuredSplices') !== expected.splices || + total(reports, 'configuredSlowReaderSplices') !== expected.slowReaders || + total(reports, 'configuredWedgedReaderSplices') !== expected.wedgedReaders || + peakActiveSplices < expected.splices - expected.wedgedReaders || + peakActiveSplices > expected.splices || + total(reports, 'completedSplices') !== expected.splices - expected.wedgedReaders || + total(reports, 'slowReaderSplicesCompleted') !== expected.slowReaders || + total(reports, 'wedgedReaderSplicesClosed') !== expected.wedgedReaders + ) throw new Error('staging mixed load evidence does not match') + if ( + total(reports, 'regionalFallbacksProved') !== 0 || + total(reports, 'oldClientUsFirstProved') !== 1 || + total(reports, 'stickyAssignmentProved') !== 1 || + total(reports, 'requestUnitInvitesOpened') !== 0 || + reports.some((report) => report.requestUnitPrincipalCount !== 0) || + total(reports, 'requestUnitCleanupProved') !== 0 || + reports.some((report) => report.requestUnitOverflowReason !== null) || + total(reports, 'rebindProbesOpened') !== 2 || + reports.some((report) => report.rebindOverflowReason !== null) + ) throw new Error('staging launch-path evidence does not match') + const readerReports = reports.filter( + (report) => report.configuredSlowReaderSplices + report.configuredWedgedReaderSplices > 0 + ) + if (expected.slowReaders > 0 && readerReports.length !== 1) { + throw new Error('staging reader pressure must have one causal owner') + } + for (const report of reports) { + const queue = report.readerQueueEvidence + if (!Array.isArray(queue)) throw new Error('staging reader queue evidence is invalid') + if (expected.slowReaders === 0 && queue.length !== 0) { + throw new Error('control load unexpectedly contains reader queue evidence') + } + const ownsReaderPressure = readerReports.includes(report) + if (expected.slowReaders > 0 && ownsReaderPressure && ( + queue.length !== 1 || + queue[0]?.origin !== 'https://c4.relay-staging.onorca.dev' || + number(queue[0]?.baselineBytes, 'reader queue baseline') > + number(queue[0]?.peakBytes, 'reader queue peak') || + number(queue[0]?.increaseBytes, 'reader queue increase') <= 0 || + queue[0].peakBytes - queue[0].baselineBytes !== queue[0].increaseBytes || + report.readerQueuedBytesPeak !== queue[0].increaseBytes + )) throw new Error('staging reader queue evidence is not causal') + if (expected.slowReaders > 0 && !ownsReaderPressure && ( + queue.length !== 0 || report.readerQueuedBytesPeak !== 0 + )) throw new Error('non-owner shard contains reader queue evidence') + } + for (const report of reports) { + if ( + number(report.generatorCpuPercent, 'generator CPU') >= GENERATOR_CPU_PERCENT_LIMIT || + number(report.generatorEventLoopP99Ms, 'generator event loop') >= + GENERATOR_EVENT_LOOP_P99_MS_LIMIT || + number(report.generatorRssGrowthMiB, 'generator RSS growth') >= + GENERATOR_RSS_GROWTH_MIB_LIMIT + ) throw new Error('staging load generator has insufficient headroom') + const shutdown = object(report.shutdownEvidence, 'load shutdown evidence') + if ( + shutdown.peerShutdowns !== report.controls || shutdown.activeControls !== 0 || + shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0 + ) throw new Error('staging load cleanup is incomplete') + } +} + +function number(value, label) { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`${label} is invalid`) + } + return value +} + +function sumMap(value, label) { + const entries = Object.entries(object(value ?? {}, label)) + return entries.reduce((total, [key, count]) => { + if (!key) throw new Error(`${label} is invalid`) + return total + number(count, label) + }, 0) +} + +function assertCoverage(timestamps, start, end, label) { + if (timestamps.length === 0) throw new Error(`${label} has no samples`) + const ordered = timestamps.map((value) => instant(value, `${label} timestamp`).valueOf()) + .sort((left, right) => left - right) + if ( + ordered[0] > start.valueOf() + MAX_LOG_EDGE_GAP_MS || + ordered.at(-1) < end.valueOf() - MAX_LOG_EDGE_GAP_MS + ) throw new Error(`${label} does not cover the canary window`) + if (ordered.some((timestamp, index) => index > 0 && timestamp - ordered[index - 1] > MAX_LOG_SAMPLE_GAP_MS)) { + throw new Error(`${label} has a sampling gap`) + } +} + +function pointValue(point) { + const value = point?.value + if (typeof value?.doubleValue === 'number') return value.doubleValue + if (typeof value?.int64Value === 'string') return Number(value.int64Value) + return NaN +} + +function cloudSqlMaximum(response, start, end) { + const points = (object(response, 'Cloud SQL response').timeSeries ?? []) + .flatMap((series) => series.points ?? []) + assertCoverage(points.map((point) => point.interval?.endTime), start, end, 'Cloud SQL metrics') + const values = points.map(pointValue) + if (values.some((value) => !Number.isFinite(value) || value < 0)) { + throw new Error('Cloud SQL backend metric is invalid') + } + return Math.max(...values) +} + +export function buildC27CanaryEvidence(input) { + const start = instant(input.startedAt, 'canary start') + const end = instant(input.endedAt, 'canary end') + if (end.valueOf() - start.valueOf() < C27_CANARY_MINIMUM_MS) { + throw new Error('C27 canary window is shorter than 5 minutes') + } + const load = object(input.loadReport, 'C27 load report') + assertC27CanaryLoad(load) + const metrics = runtimeMetrics(input.logs, start, end, C27) + assertPassingRuntimeMetrics(metrics, 'C27 canary') + metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end) + assertPassingCanary(metrics) + return { + ...baseEvidence(input, 'production', [C27]), + kind: 'production-c27-canary', + window: { startedAt: start.toISOString(), endedAt: end.toISOString() }, + load, + metrics + } +} + +function assertC27CanaryLoad(report) { + if ( + report.event !== 'relay_load_complete' || + report.controls !== 1 || report.shardCount !== 1 || report.shardIndex !== 0 || + report.relayAsiaLoadPrincipalCount !== 1 || + number(report.configuredSteadySeconds, 'canary steady seconds') < 300 || + number(report.configuredSpliceHoldSeconds, 'canary splice hold seconds') < 60 || + number(report.requiredLeaseHorizons, 'canary lease horizons') < 2 || + report.peakActive !== 1 || report.steadyMinimumActive !== 1 || + report.configuredSplices !== 1 || report.peakActiveSplices !== 1 || + report.completedSplices !== 1 || report.failedSplices !== 0 + ) throw new Error('C27 control and splice canary did not match') + for (const key of [ + 'connectionFailures', 'unexpectedCloses', 'protocolErrors', + 'refreshErrors', 'socketErrors' + ]) { + if (number(report[key], key) !== 0) throw new Error(`C27 canary ${key} must be zero`) + } + const shutdown = object(report.shutdownEvidence, 'C27 load shutdown evidence') + if ( + shutdown.peerShutdowns !== 1 || shutdown.activeControls !== 0 || + shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0 + ) throw new Error('C27 load cleanup is incomplete') +} + +function runtimeMetrics(logs, start, end, targetCellId) { + const entries = logs.map((entry) => object(entry, 'runtime metric entry')) + const directorEntries = entries.filter((entry) => entry.jsonPayload?.role === 'director') + const cellEntries = entries.filter((entry) => + entry.jsonPayload?.role === 'cell' && + entry.jsonPayload?.cellId === targetCellId && + entry.jsonPayload?.region === 'asia-east2' + ) + assertCoverage(directorEntries.map((entry) => entry.timestamp), start, end, 'director metrics') + assertCoverage(cellEntries.map((entry) => entry.timestamp), start, end, `${targetCellId} metrics`) + const identifiedDirectors = Map.groupBy( + directorEntries.filter((entry) => entry.resource?.labels?.instance_id), + (entry) => entry.resource.labels.instance_id + ) + for (const [instanceId, instanceEntries] of identifiedDirectors) { + assertCoverage(instanceEntries.map((entry) => entry.timestamp), start, end, `director ${instanceId}`) + } + const directorPayloads = directorEntries.map((entry) => entry.jsonPayload) + const cellPayloads = cellEntries.map((entry) => entry.jsonPayload) + const payloads = [...directorPayloads, ...cellPayloads] + return { + asiaSelections: directorPayloads.reduce( + (total, payload) => total + number(payload.selectedRegionsDelta?.['asia-east2'] ?? 0, 'Asia selections'), 0 + ), + regionFallbacks: directorPayloads.reduce( + (total, payload) => total + sumMap(payload.regionFallbacksDelta, 'region fallbacks'), 0 + ), + usRegionFallbacks: directorPayloads.reduce( + (total, payload) => total + number( + payload.regionFallbacksDelta?.['us-central1'] ?? 0, + 'US region fallbacks' + ), 0 + ), + unavailableRegions: directorPayloads.reduce( + (total, payload) => total + sumMap(payload.unavailableRegionsDelta, 'unavailable regions'), 0 + ), + relaySqlFailures: payloads.reduce( + (total, payload) => total + number(payload.sqlFailuresDelta, 'Relay SQL failures'), 0 + ), + databasePoolWaitingMax: Math.max(...payloads.map( + (payload) => number(payload.databasePoolWaiting, 'database pool waiting') + )), + databasePoolWaitersMax: Math.max(...payloads.map( + (payload) => number(payload.databasePoolWaitersMax, 'database pool waiters') + )), + databasePoolWaitMsMax: Math.max(...payloads.map( + (payload) => number(payload.databasePoolWaitMsMax, 'database pool wait time') + )), + targetControlsMax: Math.max(...cellPayloads.map( + (payload) => number(payload.controls, 'target controls') + )), + targetSplicesMax: Math.max(...cellPayloads.map( + (payload) => number(payload.splices, 'target splices') + )) + } +} + +function assertPassingRuntimeMetrics(metrics, label, expectedRegionFallbacks = 0) { + if (number(metrics.asiaSelections, 'Asia selections') < 1) { + throw new Error(`${label} observed no Asia selections`) + } + if ( + number(metrics.regionFallbacks, 'regionFallbacks') !== expectedRegionFallbacks || + number(metrics.usRegionFallbacks, 'usRegionFallbacks') !== expectedRegionFallbacks + ) { + throw new Error(`${label} regionFallbacks did not match the intentional probes`) + } + for (const key of [ + 'unavailableRegions', 'relaySqlFailures', 'databasePoolWaitingMax' + ]) { + if (number(metrics[key], key) !== 0) throw new Error(`${label} ${key} must be zero`) + } + if ( + number(metrics.databasePoolWaitersMax, 'databasePoolWaitersMax') > + DATABASE_POOL_TRANSIENT_WAITERS_MAX || + number(metrics.databasePoolWaitMsMax, 'databasePoolWaitMsMax') > + DATABASE_POOL_TRANSIENT_WAIT_MS_MAX + ) throw new Error(`${label} transient database pool pressure exceeded its bound`) +} + +function assertPassingCanary(metrics) { + if ( + number(metrics.targetControlsMax, 'C27 controls') < 1 || + number(metrics.targetSplicesMax, 'C27 splices') < 1 + ) throw new Error('C27 canary traffic did not reach C27') + if (number(metrics.cloudSqlBackendsMax, 'Cloud SQL backends') >= CLOUD_SQL_LIMIT) { + throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`) + } +} + +function assertProvenance(evidence, run, expected) { + const evidenceSource = object(evidence.source, 'evidence source') + if ( + run.id !== evidenceSource.runId || + run.run_attempt !== evidenceSource.runAttempt || + run.conclusion !== 'success' || + run.event !== 'workflow_dispatch' || + run.head_branch !== 'main' || + run.head_sha !== evidenceSource.commitSha || + run.repository?.full_name !== evidenceSource.repository || + run.path?.split('@')[0] !== expected.workflow || + evidenceSource.workflow !== expected.workflow || + evidenceSource.repository !== REPOSITORY || + expected.repository !== REPOSITORY || + evidenceSource.environment !== expected.environment || + evidenceSource.commitSha !== expected.commitSha + ) throw new Error('evidence workflow provenance does not match') +} + +export function verifyRolloutEvidence(evidence, run, expected) { + object(evidence, 'evidence') + object(run, 'workflow run') + if (evidence.version !== 1 || evidence.kind !== expected.kind) { + throw new Error('evidence kind is invalid') + } + assertProvenance(evidence, run, expected) + if (evidence.imageDigest !== expected.imageDigest) throw new Error('evidence image digest does not match') + if (!exactCells(evidence.topology?.cellIds, expected.cellIds)) { + throw new Error('evidence topology does not match') + } + positiveInteger(evidence.topology?.selectorGeneration, 'evidence selector generation') + if ( + expected.selectorGeneration !== undefined && + evidence.topology.selectorGeneration !== expected.selectorGeneration + ) throw new Error('evidence selector generation does not match') + const now = instant(expected.now, 'verification time') + const proofTime = instant(evidence.window?.endedAt, 'evidence time') + const maxAgeMs = expected.kind === 'staging-asia-readiness' ? 24 * 60 * 60_000 : 6 * 60 * 60_000 + if (proofTime > now || now.valueOf() - proofTime.valueOf() > maxAgeMs) { + throw new Error('rollout evidence is stale') + } + if (expected.kind === 'production-c27-canary') { + const start = instant(evidence.window?.startedAt, 'canary start') + if (proofTime.valueOf() - start.valueOf() < C27_CANARY_MINIMUM_MS) { + throw new Error('C27 canary window is shorter than 5 minutes') + } + assertC27CanaryLoad(object(evidence.load, 'canary load')) + assertPassingCanary(object(evidence.metrics, 'canary metrics')) + } + return evidence +} + +function argumentsMap(argv) { + const values = new Map() + for (let index = 1; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined || values.has(key.slice(2))) { + throw new Error('invalid rollout evidence arguments') + } + values.set(key.slice(2), value) + } + return { command: argv[0], values } +} + +function required(values, key) { + const value = values.get(key) + if (!value) throw new Error(`missing --${key}`) + return value +} + +function commonInput(values) { + return { + repository: required(values, 'repository'), runId: required(values, 'run-id'), + runAttempt: required(values, 'run-attempt'), commitSha: required(values, 'commit-sha'), + imageDigest: required(values, 'image-digest'), + selectorGeneration: required(values, 'selector-generation') + } +} + +async function main(argv) { + const { command, values } = argumentsMap(argv) + const output = required(values, 'output') + if (command === 'create-staging') { + writeFileSync(output, `${JSON.stringify(buildStagingEvidence({ + ...commonInput(values), startedAt: required(values, 'started-at'), + endedAt: required(values, 'ended-at'), + launchReport: JSON.parse(readFileSync(required(values, 'launch-report'), 'utf8')), + logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')), + cloudSql: await readCloudSqlBackends( + 'staging', required(values, 'started-at'), required(values, 'ended-at') + ) + }), null, 2)}\n`) + return + } + if (command === 'create-c27') { + const startedAt = required(values, 'started-at') + const endedAt = required(values, 'ended-at') + writeFileSync(output, `${JSON.stringify(buildC27CanaryEvidence({ + ...commonInput(values), startedAt, endedAt, + loadReport: JSON.parse(readFileSync(required(values, 'load-report'), 'utf8')), + logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')), + cloudSql: await readCloudSqlBackends('production', startedAt, endedAt) + }), null, 2)}\n`) + return + } + if (!['verify-staging', 'verify-c27'].includes(command)) throw new Error('invalid evidence command') + const kind = command === 'verify-staging' ? 'staging-asia-readiness' : 'production-c27-canary' + verifyRolloutEvidence( + JSON.parse(readFileSync(required(values, 'evidence'), 'utf8')), + JSON.parse(readFileSync(required(values, 'run-json'), 'utf8')), + { + kind, repository: REPOSITORY, + workflow: kind === 'staging-asia-readiness' ? STAGING_WORKFLOW : ADMISSION_WORKFLOW, + environment: kind === 'staging-asia-readiness' ? 'staging' : 'production', + commitSha: required(values, 'commit-sha'), imageDigest: required(values, 'image-digest'), + cellIds: [kind === 'staging-asia-readiness' ? STAGING_CELL : C27], + selectorGeneration: values.has('selector-generation') + ? positiveInteger(values.get('selector-generation'), 'selector generation') : undefined, + now: required(values, 'now') + } + ) + writeFileSync(output, 'verified\n') +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) await main(process.argv.slice(2)) diff --git a/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs b/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs new file mode 100644 index 00000000000..5d7b316605c --- /dev/null +++ b/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs @@ -0,0 +1,357 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs' +import { + buildC27CanaryEvidence, + buildStagingEvidence, + verifyRolloutEvidence +} from './relay-asia-rollout-evidence.mjs' + +const digest = `sha256:${'a'.repeat(64)}` +const commitSha = 'b'.repeat(40) +const repository = RELAY_GITHUB_REPOSITORY +const start = new Date('2026-08-13T12:00:00.000Z') +const end = new Date(start.valueOf() + 15 * 60_000) +const canaryEnd = new Date(start.valueOf() + 5 * 60_000) + +function sourceInput(overrides = {}) { + return { + repository, runId: 123, runAttempt: 2, commitSha, imageDigest: digest, + selectorGeneration: 9, ...overrides + } +} + +function metricLog(timestamp, role, cellId, overrides = {}) { + return { + timestamp: timestamp.toISOString(), + resource: { labels: role === 'director' ? { instance_id: 'director-1' } : {} }, + jsonPayload: { + role, cellId, region: role === 'cell' ? 'asia-east2' : 'us-central1', + controls: role === 'cell' ? 2_840 : 0, + splices: role === 'cell' ? 120 : 0, + selectedRegionsDelta: role === 'director' ? { 'asia-east2': 1 } : {}, + regionFallbacksDelta: {}, unavailableRegionsDelta: {}, sqlFailuresDelta: 0, + databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0, + ...overrides + } + } +} + +function completeLogs(cellId = 'production-gce-c27', windowEnd = end) { + const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 + 1 + return Array.from({ length: samples }, (_, minute) => { + const timestamp = new Date(start.valueOf() + minute * 60_000) + return [ + metricLog(timestamp, 'director', 'production-director'), + metricLog(timestamp, 'cell', cellId) + ] + }).flat() +} + +function cloudSql(max = 319, windowEnd = end) { + const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 + return { + timeSeries: [{ + points: Array.from({ length: samples }, (_, minute) => ({ + interval: { endTime: new Date(start.valueOf() + (minute + 1) * 60_000).toISOString() }, + value: { int64Value: String(minute === samples - 1 ? max : 300) } + })) + }] + } +} + +function loadReport({ + controls, shardIndex, splices = 0, slow = 0, wedged = 0, + launch = false +}) { + return { + event: 'relay_load_complete', controls, shardCount: 4, shardIndex, + configuredRampSeconds: 180, configuredSteadySeconds: 210, requiredLeaseHorizons: 2, + configuredSpliceHoldSeconds: 210, + configuredSplices: splices, configuredSlowReaderSplices: slow, + configuredWedgedReaderSplices: wedged, + peakActive: controls, steadyMinimumActive: controls, + connectionFailures: 0, rampConnectionFailures: 0, steadyConnectionFailures: 0, + transitionConnectionFailures: 0, unexpectedCloses: 0, protocolErrors: 0, + refreshErrors: 0, socketErrors: 0, failedSplices: 0, + regionalFallbacksProved: 0, + oldClientUsFirstProved: launch ? 1 : 0, + stickyAssignmentProved: launch ? 1 : 0, + requestUnitInvitesOpened: 0, + requestUnitPrincipalCount: 0, + relayAsiaLoadPrincipalCount: 32, + requestUnitOverflowReason: null, + requestUnitCleanupProved: 0, + phaseBarrierPassed: true, + rebindProbesOpened: launch ? 2 : 0, + rebindOverflowReason: null, + peakActiveSplices: splices, completedSplices: splices - wedged, + slowReaderSplicesCompleted: slow, wedgedReaderSplicesClosed: wedged, + readerQueuedBytesPeak: slow > 0 ? 1_024 : 0, + generatorCpuPercent: 25, generatorEventLoopP99Ms: 20, generatorRssGrowthMiB: 10, + readerQueueEvidence: slow > 0 ? [{ + origin: 'https://c4.relay-staging.onorca.dev', + baselineBytes: 128, + peakBytes: 1_152, + increaseBytes: 1_024 + }] : [], + shutdownEvidence: { + peerShutdowns: controls, activeControls: 0, activeSplices: 0, reconnectTimers: 0 + } + } +} + +function stagingInput(overrides = {}) { + const logs = completeLogs('staging-gce-c4') + logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 1 } + return { + ...sourceInput({ selectorGeneration: 4 }), + startedAt: start.toISOString(), endedAt: end.toISOString(), + launchReport: Array.from({ length: 4 }, (_, shardIndex) => loadReport({ + controls: 5, shardIndex, splices: 5, launch: shardIndex === 0, + slow: shardIndex === 0 ? 4 : 0, wedged: shardIndex === 0 ? 1 : 0 + })), + logs, cloudSql: cloudSql(), ...overrides + } +} + +function canaryInput(overrides = {}) { + const load = loadReport({ controls: 1, shardIndex: 0, splices: 1 }) + Object.assign(load, { + shardCount: 1, + configuredSteadySeconds: 300, + configuredSpliceHoldSeconds: 60, + relayAsiaLoadPrincipalCount: 1 + }) + return { + ...sourceInput(), startedAt: start.toISOString(), endedAt: canaryEnd.toISOString(), + loadReport: load, + logs: completeLogs('production-gce-c27', canaryEnd), cloudSql: cloudSql(319, canaryEnd), + ...overrides + } +} + +function workflowRun(evidence, overrides = {}) { + return { + id: evidence.source.runId, run_attempt: evidence.source.runAttempt, + conclusion: 'success', event: 'workflow_dispatch', head_branch: 'main', + head_sha: evidence.source.commitSha, + repository: { full_name: evidence.source.repository }, path: evidence.source.workflow, + ...overrides + } +} + +function verifyExpected(kind, overrides = {}) { + const staging = kind === 'staging-asia-readiness' + return { + kind, repository, + workflow: staging + ? relayWorkflowPath('prove-relay-asia-staging.yml') + : relayWorkflowPath('operate-relay-asia-admission.yml'), + environment: staging ? 'staging' : 'production', commitSha, imageDigest: digest, + cellIds: [staging ? 'staging-gce-c4' : 'production-gce-c27'], + now: new Date(end.valueOf() + 60_000).toISOString(), ...overrides + } +} + +test('accepts the exact sharded staging load, telemetry, and provenance proof', () => { + const evidence = buildStagingEvidence(stagingInput()) + assert.equal(evidence.load.launch.controls, 20) + assert.equal(evidence.load.launch.peakActiveSplices, 20) + assert.equal(evidence.load.launch.readerQueueEvidence.length, 1) + assert.equal(verifyRolloutEvidence( + evidence, workflowRun(evidence), verifyExpected('staging-asia-readiness') + ), evidence) +}) + +test('ignores unrelated legacy cell metrics outside the Asia proof', () => { + const input = stagingInput() + const legacy = metricLog(start, 'cell', 'staging-gce-c1', { + region: undefined, + sqlFailuresDelta: 1 + }) + delete legacy.jsonPayload.databasePoolWaiting + delete legacy.jsonPayload.databasePoolWaitersMax + delete legacy.jsonPayload.databasePoolWaitMsMax + input.logs.push(legacy) + + assert.equal(buildStagingEvidence(input).metrics.relaySqlFailures, 0) +}) + +test('accepts bounded transient pool waits without a sampled queue', () => { + const input = stagingInput() + input.logs[0].jsonPayload.databasePoolWaitersMax = 4 + input.logs[0].jsonPayload.databasePoolWaitMsMax = 50 + + const metrics = buildStagingEvidence(input).metrics + assert.equal(metrics.databasePoolWaitingMax, 0) + assert.equal(metrics.databasePoolWaitersMax, 4) + assert.equal(metrics.databasePoolWaitMsMax, 50) +}) + +test('requires exactly the intentional sticky-assignment fallback in staging', () => { + const missing = stagingInput() + missing.logs[0].jsonPayload.regionFallbacksDelta = {} + assert.throws(() => buildStagingEvidence(missing), /intentional probes/) + + const extra = stagingInput() + extra.logs[2].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } + assert.throws(() => buildStagingEvidence(extra), /intentional probes/) + + const substituted = stagingInput() + substituted.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } + assert.throws(() => buildStagingEvidence(substituted), /intentional probes/) +}) + +test('accepts the wedged splice outside the sustained non-wedged peak', () => { + const input = stagingInput() + input.launchReport[0].peakActiveSplices = 4 + input.logs.filter((entry) => entry.jsonPayload.role === 'cell') + .forEach((entry) => { entry.jsonPayload.splices = 19 }) + const evidence = buildStagingEvidence(input) + assert.equal(evidence.load.launch.peakActiveSplices, 19) + + input.launchReport[0].peakActiveSplices = 3 + assert.throws(() => buildStagingEvidence(input), /mixed load evidence/) +}) + +test('rejects staging evidence with incomplete load or cleanup', () => { + const input = stagingInput() + input.launchReport[0].steadyMinimumActive-- + assert.throws(() => buildStagingEvidence(input), /required controls/) + const cleanup = stagingInput() + cleanup.launchReport[0].shutdownEvidence.activeControls = 1 + assert.throws(() => buildStagingEvidence(cleanup), /cleanup/) + const nonCausal = stagingInput() + nonCausal.launchReport[0].readerQueueEvidence[0].increaseBytes = 0 + assert.throws(() => buildStagingEvidence(nonCausal), /not causal/) + const multipleOwners = stagingInput() + multipleOwners.launchReport[0].configuredSlowReaderSplices-- + multipleOwners.launchReport[0].slowReaderSplicesCompleted-- + multipleOwners.launchReport[1] = loadReport({ + controls: 5, shardIndex: 1, splices: 5, slow: 1 + }) + assert.throws(() => buildStagingEvidence(multipleOwners), /one causal owner/) + const overloaded = stagingInput() + overloaded.launchReport[0].generatorCpuPercent = 80 + assert.throws(() => buildStagingEvidence(overloaded), /insufficient headroom/) + const missingLaunchProof = stagingInput() + missingLaunchProof.launchReport[0].rebindProbesOpened = 0 + assert.throws(() => buildStagingEvidence(missingLaunchProof), /launch-path/) + const offTarget = stagingInput() + offTarget.logs.filter((entry) => entry.jsonPayload.role === 'cell') + .forEach((entry) => { entry.jsonPayload.controls = 19 }) + assert.throws(() => buildStagingEvidence(offTarget), /did not reach C4/) +}) + +test('rejects staging evidence with a shortened splice hold', () => { + const input = stagingInput() + input.launchReport[0].configuredSpliceHoldSeconds = 60 + assert.throws(() => buildStagingEvidence(input), /load profile does not match/) +}) + +for (const [label, value] of [['missing', undefined], ['malformed', 'invalid']]) { + test(`rejects staging evidence with a ${label} splice hold`, () => { + const input = stagingInput() + input.launchReport[0].configuredSpliceHoldSeconds = value + assert.throws(() => buildStagingEvidence(input), /staging splice hold seconds is invalid/) + }) +} + +for (const [label, mutate, message] of [ + ['phase barrier', (input) => { input.launchReport[0].phaseBarrierPassed = false }, /profile/], + ['old-client routing', (input) => { input.launchReport[0].oldClientUsFirstProved = 0 }, /launch-path/], + ['sticky routing', (input) => { input.launchReport[0].stickyAssignmentProved = 0 }, /launch-path/] +]) { + test(`rejects staging evidence without ${label} proof`, () => { + const input = stagingInput() + mutate(input) + assert.throws(() => buildStagingEvidence(input), message) + }) +} + +test('rejects staging evidence from a non-canonical repository', () => { + assert.throws(() => buildStagingEvidence(stagingInput({ repository: 'fork/orca-cloud' })), /repository/) +}) + +test('rejects mismatched staging provenance, digest, topology, or age', () => { + const evidence = buildStagingEvidence(stagingInput()) + const expected = verifyExpected('staging-asia-readiness') + assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence, { + head_sha: 'c'.repeat(40) + }), expected), /provenance/) + assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), { + ...expected, commitSha: 'c'.repeat(40) + }), /provenance/) + assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), { + ...expected, imageDigest: `sha256:${'c'.repeat(64)}` + }), /image digest/) + assert.throws(() => verifyRolloutEvidence({ + ...evidence, topology: { ...evidence.topology, cellIds: ['staging-gce-c3'] } + }, workflowRun(evidence), expected), /topology/) + assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), { + ...expected, now: new Date(end.valueOf() + 25 * 60 * 60_000).toISOString() + }), /stale/) +}) + +test('builds and verifies a passing continuous 5-minute C27 canary', () => { + const evidence = buildC27CanaryEvidence(canaryInput()) + assert.equal(evidence.load.completedSplices, 1) + assert.equal(evidence.metrics.asiaSelections, 6) + assert.equal(evidence.metrics.cloudSqlBackendsMax, 319) + assert.equal(verifyRolloutEvidence( + evidence, workflowRun(evidence), + verifyExpected('production-c27-canary', { selectorGeneration: 9 }) + ), evidence) +}) + +test('rejects short, sparse, or unrelated-cell-only C27 coverage', () => { + assert.throws(() => buildC27CanaryEvidence(canaryInput({ + endedAt: new Date(canaryEnd.valueOf() - 1).toISOString() + })), /shorter than 5 minutes/) + const sparse = completeLogs('production-gce-c27', canaryEnd).filter((entry) => + entry.timestamp === start.toISOString() || entry.timestamp === canaryEnd.toISOString() + ) + assert.throws(() => buildC27CanaryEvidence(canaryInput({ logs: sparse })), /sampling gap/) + assert.throws(() => buildC27CanaryEvidence(canaryInput({ + logs: completeLogs('production-gce-c26', canaryEnd) + })), /production-gce-c27 metrics has no samples/) +}) + +test('rejects a C27 canary without a real control and splice', () => { + const input = canaryInput() + input.loadReport.completedSplices = 0 + assert.throws(() => buildC27CanaryEvidence(input), /did not match/) + const shortHold = canaryInput() + shortHold.loadReport.configuredSpliceHoldSeconds = 59 + assert.throws(() => buildC27CanaryEvidence(shortHold), /did not match/) +}) + +for (const [label, mutation, message] of [ + ['Asia selections', (input) => input.logs.forEach((entry) => { entry.jsonPayload.selectedRegionsDelta = {} }), /no Asia selections/], + ['region fallbacks', (input) => { input.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } }, /regionFallbacks/], + ['unavailable regions', (input) => { input.logs[0].jsonPayload.unavailableRegionsDelta = { 'asia-east2': 1 } }, /unavailableRegions/], + ['Relay SQL failures', (input) => { input.logs[0].jsonPayload.sqlFailuresDelta = 1 }, /relaySqlFailures/], + ['pool waiting', (input) => { input.logs[0].jsonPayload.databasePoolWaiting = 1 }, /databasePoolWaitingMax/], + ['pool waiters', (input) => { input.logs[0].jsonPayload.databasePoolWaitersMax = 5 }, /transient database pool pressure/], + ['pool wait time', (input) => { input.logs[0].jsonPayload.databasePoolWaitMsMax = 51 }, /transient database pool pressure/], + ['C27 controls', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell') + .forEach((entry) => { entry.jsonPayload.controls = 0 }), /did not reach C27/], + ['C27 splices', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell') + .forEach((entry) => { entry.jsonPayload.splices = 0 }), /did not reach C27/], + ['Cloud SQL headroom', (input) => { input.cloudSql = cloudSql(320, canaryEnd) }, /below 320/] +]) { + test(`rejects C27 evidence with ${label}`, () => { + const input = canaryInput() + mutation(input) + assert.throws(() => buildC27CanaryEvidence(input), message) + }) +} + +test('rejects C27 evidence from a different selector generation', () => { + const evidence = buildC27CanaryEvidence(canaryInput()) + assert.throws(() => verifyRolloutEvidence( + evidence, workflowRun(evidence), + verifyExpected('production-c27-canary', { selectorGeneration: 10 }) + ), /selector generation/) +}) diff --git a/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs b/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs new file mode 100644 index 00000000000..965a3ce142c --- /dev/null +++ b/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { relayWorkflowUrl } from './relay-repository.mjs' + +const workflow = readFileSync( + relayWorkflowUrl('deploy-relay-asia-topology.yml'), + 'utf8' +) +const iam = readFileSync( + new URL('../../infra/terraform/relay-asia-topology-iam.tf', import.meta.url), + 'utf8' +) +const cells = readFileSync( + new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), + 'utf8' +) +const variables = readFileSync( + new URL('../../infra/terraform/variables.tf', import.meta.url), + 'utf8' +) + +test('uses only its exact workflow-bound topology identity', () => { + assert.match(workflow, /production-cloud-sql-rollout/) + assert.match(workflow, /relay-staging-mutation/) + assert.match(workflow, /RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(workflow, /RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT/) + assert.doesNotMatch(workflow, /GCP_DEPLOY_SERVICE_ACCOUNT/) + assert.match( + iam, + /assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_topology_workflow_file\}@refs\/heads\/main'/ + ) + assert.match(iam, /assertion\.ref == 'refs\/heads\/main'/) + assert.match(iam, /assertion\.event_name == 'workflow_dispatch'/) + assert.match(iam, /assertion\.environment == '\$\{var\.environment\}'/) +}) + +test('plans only additive Asia topology and applies the saved plan', () => { + assert.equal((workflow.match(/manage_artifact_dns=false/g) ?? []).length, 2) + for (const target of [ + 'relay_gce_additional', + 'google_compute_instance_template.relay_gce_cell', + 'google_compute_instance_group_manager.relay_gce_cell', + 'google_compute_backend_service.relay_gce_cell', + 'google_compute_url_map.relay_gce' + ]) assert.match(workflow, new RegExp(target.replaceAll('.', '\\.'))) + assert.match(workflow, /apply -input=false -auto-approve "\$\{\{ steps\.plan\.outputs\.plan \}\}"/) + assert.match(workflow, /prepare-relay-asia-topology-input\.mjs/) + assert.doesNotMatch(workflow, /steps\.variables\.outputs\.file/) + assert.match(workflow, /\.variables\.relay_gce_cells\.value/) + assert.match( + workflow, + /\.variables\.relay_gce_additional_region_subnetwork_cidrs\.value/ + ) + assert.doesNotMatch(workflow, /terraform -chdir=infra\/terraform console/) + assert.equal((workflow.match(/-var-file="\$\{TF_VARS\}"/g) ?? []).length, 2) + assert.doesNotMatch(workflow, /terraform[^\n]*apply[^\n]*-target/) + assert.doesNotMatch(workflow, /google_(?:sql|cloudflare|dns|certificate_manager)/) +}) + +test('validates before apply and proves convergence afterward', () => { + assert.equal((workflow.match(/validate-relay-asia-topology-plan\.mjs/g) ?? []).length, 2) + assert.match(workflow, /APPLY_RELAY_ASIA_TOPOLOGY/) + assert.match(workflow, /test "\$\(jq -er '\.changes'/) + assert.match(workflow, /Register the exact new cells atomically as migration-only/) +}) + +test('checks the connection budget and production live ceiling before planning', () => { + assert.match(workflow, /relay-cloud-sql-connection-budget\.mjs/) + assert.match(workflow, /gcloud sql instances describe "\$\{CLOUD_SQL_INSTANCE\}"/) + assert.match(workflow, /select\(\.name == "max_connections"\)/) + assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360/) + assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17/) + assert.match(workflow, /live_source=verified-shape-default/) + assert.match(workflow, /test "\$\(jq -er '\.settings\.tier'/) + assert.match(workflow, /test "\$\(jq -er '\.databaseVersion'/) + assert.match(workflow, /test "\$\{live_max\}" = "\$\{checked_max\}"/) + assert.ok( + workflow.indexOf('relay-cloud-sql-connection-budget.mjs') < + workflow.indexOf('terraform -chdir=infra/terraform plan') + ) +}) + +test('binds computed Asia references to the matching Terraform cell resources', () => { + assert.match(cells, /instance_template = google_compute_instance_template\.relay_gce_cell\[each\.key\]\.self_link/) + assert.match(cells, /group\s+= google_compute_instance_group_manager\.relay_gce_cell\[each\.key\]\.instance_group/) + assert.match(cells, /default_service = google_compute_backend_service\.relay_gce_cell\[cell\.key\]\.id/) + assert.match(cells, /subnetwork = local\.relay_gce_subnetworks\[each\.value\.region\]/) +}) + +test('keeps cross-variable region constraints in Terraform 1.5 check blocks', () => { + assert.doesNotMatch(variables, /region != var\.region/) + assert.doesNotMatch(variables, /cell\.region == var\.region/) + assert.match(cells, /check "relay_gce_fixed_one_topology"[\s\S]*?region != var\.region/) + assert.match(cells, /cell\.region == var\.region[\s\S]*?configured subnetwork/) +}) + +test('the custom role cannot delete topology or mutate SQL and DNS', () => { + assert.doesNotMatch(iam, /compute\.[A-Za-z]+\.delete/) + assert.doesNotMatch( + iam, + /roles\/viewer|cloudsql\.instances\.(?:update|delete)|dns\.|certificatemanager|cloudflare/i + ) + assert.match(iam, /resource "google_project_iam_custom_role" "github_relay_asia_topology_read"/) + assert.match(iam, /"cloudsql\.instances\.get"/) + assert.match(iam, /"run\.revisions\.get"/) + assert.match(iam, /"run\.services\.get"/) + assert.match(iam, /"serviceusage\.services\.list"/) + assert.match(iam, /"compute\.networks\.updatePolicy"/) + assert.match(iam, /"compute\.healthChecks\.useReadOnly"/) + assert.match(iam, /"compute\.instanceGroups\.create"/) + assert.match(iam, /"compute\.instances\.use"/) + assert.match(iam, /roles\/storage\.objectAdmin/) + assert.match(iam, /default\.tfstate/) + assert.match(iam, /default\.tflock/) + assert.match( + iam, + /resource "google_project_iam_custom_role" "github_relay_asia_topology_state_list"[\s\S]*?permissions = \["storage\.objects\.list"\]/ + ) + assert.match( + iam, + /resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state_list"[\s\S]*?role\s+= google_project_iam_custom_role\.github_relay_asia_topology_state_list\[0\]\.id/ + ) +}) diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs new file mode 100644 index 00000000000..79036918f23 --- /dev/null +++ b/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs @@ -0,0 +1,148 @@ +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +const DEFAULT_PATHS = { + productionTfvars: new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), + terraformVariables: new URL('../../infra/terraform/variables.tf', import.meta.url), + relayConfig: new URL('../../apps/relay/src/config.ts', import.meta.url) +} + +// Auth and API live outside the Relay tree, so their consumption is published as a contract +// rather than parsed from their source. production-cloud-sql-app-consumers.test.mjs binds it back. +const APP_CONSUMERS_CONTRACT = new URL( + '../contracts/production-cloud-sql-app-consumers.json', + import.meta.url +) + +const APP_CONSUMER_FIELDS = ['authInstances', 'authPoolMax', 'apiInstances', 'apiPoolMax', 'maxConnections'] + +export function readProductionCloudSqlAppConsumers(contract) { + const parsed = contract ?? JSON.parse(readFileSync(APP_CONSUMERS_CONTRACT, 'utf8')) + for (const field of APP_CONSUMER_FIELDS) { + const value = parsed[field] + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`could not read ${field} from the app consumer contract`) + } + } + return parsed +} + +function requiredInteger(source, pattern, label) { + const value = Number(source.match(pattern)?.[1]) + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`could not read ${label}`) + return value +} + +function productionCells(source, defaultPoolMax) { + const fencedMatch = source.match(/relay_gce_fenced_cells\s*=\s*\[([^\]]*)\]/) + if (!fencedMatch) throw new Error('could not read fenced Relay cells') + const fenced = new Set([...fencedMatch[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])) + const cells = [...source.matchAll(/"(production-gce-[^"]+)"\s*=\s*\{([\s\S]*?)\n\s*\}/g)].map( + ([, id, body]) => ({ + id, + fenced: fenced.has(id), + region: body.match(/\bregion\s*=\s*"([^"]+)"/)?.[1] ?? 'us-central1', + poolMax: body.match(/\bdatabase_pool_max\s*=\s*(\d+)/) + ? Number(body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)[1]) + : defaultPoolMax + }) + ) + if (cells.length === 0) throw new Error('could not read production Relay cells') + return cells +} + +export function calculateRelayCloudSqlConnectionBudget(inputs) { + const consumers = { + cells: inputs.cellPoolTotal + inputs.asiaCellCount * inputs.asiaPoolMax, + directors: inputs.directorInstances * inputs.directorPoolMax, + auth: inputs.authInstances * inputs.authPoolMax, + api: inputs.apiInstances * inputs.apiPoolMax + } + const configuredMaximum = Object.values(consumers).reduce((total, value) => total + value, 0) + const retainedDirectorRollback = inputs.directorInstances * inputs.directorPoolMax + const candidateOverlap = { + relayDirectorCandidate: retainedDirectorRollback * 2, + apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax, + authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax, + relayCells: retainedDirectorRollback + } + const rolloutOverlap = Math.max(...Object.values(candidateOverlap)) + const operatingMaximum = configuredMaximum + rolloutOverlap + inputs.maintenanceAdminAllowance + const usableCeiling = inputs.maxConnections - inputs.explicitReserve + const budgetedTotal = operatingMaximum + inputs.explicitReserve + return { + maxConnections: inputs.maxConnections, + consumers, + asia: { cells: inputs.asiaCellCount, poolMax: inputs.asiaPoolMax }, + configuredMaximum, + rolloutOverlap: { + ...candidateOverlap, + retainedDirectorRollback, + maximum: rolloutOverlap, + reason: 'serialized rollouts include directly addressable tagged revisions outside service-level caps' + }, + maintenanceAdminAllowance: inputs.maintenanceAdminAllowance, + maintenanceAdminAllowanceReason: 'covers bounded work outside configured services', + explicitReserve: inputs.explicitReserve, + explicitReserveReason: 'remains unavailable to configured services and planned rollouts', + usableCeiling, + operatingMaximum, + remainingWithinUsableCeiling: usableCeiling - operatingMaximum, + budgetedTotal, + unallocated: inputs.maxConnections - budgetedTotal, + withinBudget: operatingMaximum <= usableCeiling && budgetedTotal < inputs.maxConnections + } +} + +export function readRelayCloudSqlConnectionBudget({ + sources, + appConsumers, + proposedAsiaCellCount = 3, + asiaPoolMax = 10, + maxConnections, + maintenanceAdminAllowance = 5, + explicitReserve = 10 +} = {}) { + const read = (name) => sources?.[name] ?? readFileSync(DEFAULT_PATHS[name], 'utf8') + const apps = readProductionCloudSqlAppConsumers(appConsumers) + const productionTfvars = read('productionTfvars') + const terraformVariables = read('terraformVariables') + const relayConfig = read('relayConfig') + const cells = productionCells( + productionTfvars, + requiredInteger(relayConfig, /RELAY_DATABASE_POOL_MAX\s*=\s*(\d+)/, 'Relay pool maximum') + ) + const poweredCells = cells.filter(({ fenced }) => !fenced) + const configuredAsiaCells = poweredCells.filter(({ region }) => region === 'asia-east2') + const nonAsiaCells = poweredCells.filter(({ region }) => region !== 'asia-east2') + const cellPoolTotal = nonAsiaCells.reduce((total, cell) => total + cell.poolMax, 0) + const asiaCellCount = configuredAsiaCells.length || proposedAsiaCellCount + const configuredAsiaPoolMax = configuredAsiaCells[0]?.poolMax ?? asiaPoolMax + if (configuredAsiaCells.some(({ poolMax }) => poolMax !== configuredAsiaPoolMax)) { + throw new Error('Asia Relay cells must use one checked pool maximum') + } + return calculateRelayCloudSqlConnectionBudget({ + cellPoolTotal, + asiaCellCount, + asiaPoolMax: configuredAsiaPoolMax, + directorInstances: requiredInteger(productionTfvars, /relay_max_instances\s*=\s*(\d+)/, 'director instances'), + directorPoolMax: requiredInteger( + terraformVariables, + /variable\s+"relay_director_database_pool_max"[\s\S]*?default\s*=\s*(\d+)/, + 'director pool maximum' + ), + authInstances: apps.authInstances, + authPoolMax: apps.authPoolMax, + apiInstances: apps.apiInstances, + apiPoolMax: apps.apiPoolMax, + maxConnections: maxConnections ?? apps.maxConnections, + maintenanceAdminAllowance, + explicitReserve + }) +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const report = readRelayCloudSqlConnectionBudget() + console.log(JSON.stringify({ event: 'relay_cloud_sql_connection_budget', ...report }, null, 2)) + if (!report.withinBudget) process.exitCode = 1 +} diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs new file mode 100644 index 00000000000..a26d24c274d --- /dev/null +++ b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs @@ -0,0 +1,110 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { + calculateRelayCloudSqlConnectionBudget, + readRelayCloudSqlConnectionBudget +} from './relay-cloud-sql-connection-budget.mjs' + +test('production plus three Asia pools preserves allowance and reserve below the ceiling', () => { + const report = readRelayCloudSqlConnectionBudget() + + assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 }) + assert.deepEqual(report.asia, { cells: 3, poolMax: 10 }) + assert.equal(report.configuredMaximum, 315) + assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30) + assert.equal(report.rolloutOverlap.apiCandidate, 65) + assert.equal(report.rolloutOverlap.authCandidate, 35) + assert.equal(report.rolloutOverlap.relayCells, 15) + assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15) + assert.equal(report.rolloutOverlap.maximum, 65) + assert.equal(report.maintenanceAdminAllowance, 5) + assert.equal(report.explicitReserve, 10) + assert.equal(report.usableCeiling, 390) + assert.equal(report.operatingMaximum, 385) + assert.equal(report.remainingWithinUsableCeiling, 5) + assert.equal(report.budgetedTotal, 395) + assert.equal(report.unallocated, 5) + assert.equal(report.withinBudget, true) +}) + +test('fails closed when pool growth consumes the explicit reserve', () => { + const report = calculateRelayCloudSqlConnectionBudget({ + cellPoolTotal: 200, + asiaCellCount: 3, + asiaPoolMax: 20, + directorInstances: 5, + directorPoolMax: 3, + authInstances: 2, + authPoolMax: 10, + apiInstances: 20, + apiPoolMax: 5, + maxConnections: 400, + maintenanceAdminAllowance: 5, + explicitReserve: 10 + }) + + assert.equal(report.operatingMaximum, 515) + assert.equal(report.withinBudget, false) +}) + +test('excludes fenced cell pools and reads per-cell pool overrides', () => { + const report = readRelayCloudSqlConnectionBudget({ + proposedAsiaCellCount: 1, + appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 }, + sources: { + productionTfvars: ` + relay_max_instances = 1 + relay_gce_fenced_cells = ["production-gce-c1"] + relay_gce_cells = { + "production-gce-c1" = { database_pool_max = 99 + } + "production-gce-c2" = { database_pool_max = 4 + } + } + `, + terraformVariables: 'variable "relay_director_database_pool_max" { default = 3 }', + relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10' + }, + maxConnections: 100, + maintenanceAdminAllowance: 1, + explicitReserve: 1 + }) + + assert.equal(report.consumers.cells, 14) + assert.equal(report.operatingMaximum, 46) + assert.equal(report.budgetedTotal, 47) +}) + +test('requires strict headroom below the physical ceiling', () => { + const report = calculateRelayCloudSqlConnectionBudget({ + cellPoolTotal: 20, + asiaCellCount: 0, + asiaPoolMax: 10, + directorInstances: 1, + directorPoolMax: 3, + authInstances: 1, + authPoolMax: 10, + apiInstances: 1, + apiPoolMax: 5, + maxConnections: 50, + maintenanceAdminAllowance: 9, + explicitReserve: 3 + }) + + assert.equal(report.budgetedTotal, 63) + assert.equal(report.withinBudget, false) +}) + +test('pages Relay channels when Cloud SQL backends consume headroom', () => { + const terraform = readFileSync( + new URL('../../infra/terraform/relay-observability.tf', import.meta.url), + 'utf8' + ) + const policy = terraform.match( + /resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" \{([\s\S]*?)\n\}/ + )?.[1] + + assert.ok(policy) + assert.match(policy, /notification_channels\s*=\s*var\.relay_alert_notification_channels/) +}) diff --git a/cloud/dev/scripts/relay-gce-terraform-fence.mjs b/cloud/dev/scripts/relay-gce-terraform-fence.mjs new file mode 100644 index 00000000000..82b9dcbff6c --- /dev/null +++ b/cloud/dev/scripts/relay-gce-terraform-fence.mjs @@ -0,0 +1,1387 @@ +import { execFileSync } from 'node:child_process' +import { createHash, randomUUID } from 'node:crypto' +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + statSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const MIG_ADDRESS_PREFIX = 'google_compute_instance_group_manager.relay_gce_cell' +const PLAN_OBJECT_PREFIX = 'terraform/state/relay-fence-plans' +const TERRAFORM_STATE_LINEAGE = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i +const GENERATED_MIG_VERSION_NAME = + /^0\/[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?\+00:00$/ + +function changedActions(change) { + return change.change.actions.filter((action) => action !== 'no-op' && action !== 'read') +} + +export function validateTerraformFencePlan(plan, expected) { + const changes = (plan.resource_changes ?? []).filter( + (change) => changedActions(change).length > 0 + ) + if (changes.length !== 1) throw new Error('fence plan must contain exactly one mutation') + const [change] = changes + if (change.address !== `${MIG_ADDRESS_PREFIX}["${expected.cellId}"]`) { + throw new Error('fence plan mutates an unexpected resource') + } + if ( + JSON.stringify(change.change.actions) !== JSON.stringify(['update']) || + Number(change.change.before?.target_size) !== 1 || + Number(change.change.after?.target_size) !== 0 + ) { + throw new Error('fence plan must update only the requested MIG from one to zero') + } + for (const field of ['name', 'zone', 'instance_group']) { + if ( + change.change.before?.[field] !== change.change.after?.[field] || + change.change.after?.[field] !== expected[field] + ) { + throw new Error(`fence plan changed or mismatched MIG ${field}`) + } + } + const beforeGeneration = change.change.before?.version?.[0]?.instance_template + const afterGeneration = change.change.after?.version?.[0]?.instance_template + if ( + beforeGeneration !== afterGeneration || + afterGeneration !== expected.generationIdentity + ) { + throw new Error('fence plan changed or mismatched the MIG generation') + } + return change +} + +export function validateTerraformFenceCompletionPlan(plan, expected) { + const changes = (plan.resource_changes ?? []).filter( + (change) => changedActions(change).length > 0 + ) + if (changes.length === 0) return + if (changes.length !== 1) { + throw new Error('completed fence plan contains an unexpected mutation') + } + const [change] = changes + const before = change.change.before + const after = change.change.after + const beforeVersion = before?.version?.[0] + const afterVersion = after?.version?.[0] + if ( + change.address !== `${MIG_ADDRESS_PREFIX}["${expected.cellId}"]` || + JSON.stringify(change.change.actions) !== JSON.stringify(['update']) || + Number(before?.target_size) !== 0 || + Number(after?.target_size) !== 0 || + before?.name !== expected.name || + after?.name !== expected.name || + before?.zone !== expected.zone || + after?.zone !== expected.zone || + before?.instance_group !== expected.instance_group || + after?.instance_group !== expected.instance_group || + before?.version?.length !== 1 || + after?.version?.length !== 1 || + beforeVersion?.instance_template !== expected.generationIdentity || + afterVersion?.instance_template !== expected.generationIdentity || + !GENERATED_MIG_VERSION_NAME.test(beforeVersion?.name ?? '') || + afterVersion?.name !== 'primary' + ) { + throw new Error('completed fence plan is not a safe provider normalization') + } + const normalizedBefore = structuredClone(before) + normalizedBefore.version[0].name = afterVersion.name + if (JSON.stringify(normalizedBefore) !== JSON.stringify(after)) { + throw new Error('completed fence plan changes more than the provider version label') + } + return change +} + +export function terraformFenceState(state, expected) { + const resources = state.values?.root_module?.resources ?? [] + const matches = resources.filter( + (resource) => resource.address === `${MIG_ADDRESS_PREFIX}["${expected.cellId}"]` + ) + if (matches.length !== 1) throw new Error('Terraform state has no unique requested MIG') + const values = matches[0].values + if ( + values.name !== expected.name || + values.zone !== expected.zone || + values.instance_group !== expected.instance_group || + values.version?.[0]?.instance_template !== expected.generationIdentity + ) { + throw new Error('Terraform state MIG identity does not match the reviewed topology') + } + const targetSize = Number(values.target_size) + if (![0, 1].includes(targetSize)) throw new Error('Terraform state MIG size is unsafe') + return targetSize +} + +export function classifyTerraformFenceProgress({ + stateTargetSize, + liveTargetSize, + instanceCount, + operationStatus, + operationError = false, + operationAuditBound = false +}) { + if (operationError) throw new Error('Terraform fence GCE operation failed') + if (operationStatus === 'DONE' && !operationAuditBound) { + throw new Error('Terraform fence GCE operation lacks exact audit binding') + } + if ( + stateTargetSize === 0 && + liveTargetSize === 0 && + instanceCount === 0 && + operationStatus === 'DONE' + ) { + return 'complete' + } + if ( + [0, 1].includes(stateTargetSize) && + [0, 1].includes(liveTargetSize) && + ['PENDING', 'RUNNING'].includes(operationStatus) + ) { + return 'in-progress' + } + if ( + stateTargetSize === 1 && + liveTargetSize === 0 && + instanceCount === 0 && + operationStatus === 'DONE' + ) { + return 'reconcile-state' + } + if ( + stateTargetSize === 1 && + liveTargetSize === 1 && + instanceCount === 1 && + operationStatus === 'ABSENT' + ) { + return 'not-started' + } + throw new Error('Terraform fence progress is ambiguous or unsafe') +} + +function sha256(path, readFile) { + return createHash('sha256').update(readFile(path)).digest('hex') +} + +function terraformJson(deps, args) { + return JSON.parse(deps.terraform(args, { encoding: 'utf8' })) +} + +export function terraformProcessStdio(options = {}) { + if (!options.encoding) return 'inherit' + return [options.input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'] +} + +function defaultTerraform(args, options = {}) { + return execFileSync(process.env.IAC_TOOL || 'terraform', args, { + ...options, + stdio: terraformProcessStdio(options) + }) +} + +function defaultGit(args, options = {}) { + return execFileSync('git', args, { + ...options, + stdio: options.encoding ? ['ignore', 'pipe', 'pipe'] : 'inherit' + }) +} + +function privatePlanDirectory(deps) { + const previousMask = process.umask(0o077) + try { + const directory = deps.mkdtemp(join(deps.tmpdir(), 'orca-relay-fence-')) + deps.chmod(directory, 0o700) + return directory + } finally { + process.umask(previousMask) + } +} + +function exactMigExpected(cell) { + return { + cellId: cell.cellId, + name: cell.migName, + zone: cell.zone, + instance_group: cell.instanceGroup, + generationIdentity: cell.generationIdentity + } +} + +function assertFenceIdentity(cell) { + if (!cell.generationIdentity) throw new Error('requested cell has no generation identity') +} + +function assertAttemptMatches(config, attempt, requirePlanGeneration = true) { + for (const [field, expected] of Object.entries({ + environment: config.environment, + cellId: config.cell.cellId, + cellIncarnation: config.cellIncarnation, + migName: config.cell.migName, + instanceGroup: config.cell.instanceGroup, + generationIdentity: config.cell.generationIdentity, + fenceCommit: config.fenceCommit + })) { + if (attempt[field] !== expected) throw new Error(`fence attempt ${field} mismatch`) + } + if (!/^[a-f0-9]{64}$/.test(attempt.planSha256 ?? '')) { + throw new Error('fence attempt has no valid saved-plan digest') + } + if ( + attempt.planObjectName !== + `${PLAN_OBJECT_PREFIX}/${config.environment}/${attempt.attemptId}.tfplan` + ) { + throw new Error('fence attempt saved-plan object mismatch') + } + if ( + requirePlanGeneration && + !/^[1-9][0-9]{0,30}$/.test(attempt.planObjectGeneration ?? '') + ) { + throw new Error('fence attempt has no valid saved-plan generation') + } + if (!/^[a-f0-9]{64}$/.test(attempt.varFileSha256 ?? '')) { + throw new Error('fence attempt has no valid variable-file digest') + } + if ( + !TERRAFORM_STATE_LINEAGE.test(attempt.terraformStateLineage ?? '') || + !Number.isSafeInteger(attempt.terraformStateSerial) || + attempt.terraformStateSerial < 0 + ) { + throw new Error('fence attempt has no valid Terraform state identity') + } + if ( + !/^[1-9][0-9]{0,30}$/.test( + attempt.terraformStateObjectGeneration ?? '' + ) || + !/^[a-f0-9]{64}$/.test(attempt.terraformStateObjectSha256 ?? '') + ) { + throw new Error('fence attempt has no valid Terraform state object binding') + } + if (attempt.requestReason !== `orca-relay-fence/${attempt.attemptId}`) { + throw new Error('fence attempt request-reason mismatch') + } +} + +export function terraformFenceStateIdentity(config, deps = {}) { + const terraform = deps.terraform ?? defaultTerraform + const state = terraformJson({ terraform }, [ + `-chdir=${config.terraformDir}`, + 'state', + 'pull' + ]) + if ( + !TERRAFORM_STATE_LINEAGE.test(state.lineage ?? '') || + !Number.isSafeInteger(state.serial) || + state.serial < 0 + ) { + throw new Error('Terraform state has no valid lineage or serial') + } + return { lineage: state.lineage, serial: state.serial } +} + +export function assertReviewedFenceCheckout(config, deps = {}) { + const environment = deps.environment ?? process.env + const git = deps.git ?? defaultGit + const readFile = deps.readFile ?? readFileSync + const varPath = join(config.terraformDir, config.varFile) + const imageCommit = environment.ORCA_RELAY_FENCE_IMAGE_COMMIT + if (imageCommit !== undefined) { + if (!/^[a-f0-9]{40}$/.test(imageCommit) || imageCommit !== config.fenceCommit) { + throw new Error('fence commit does not match immutable broker image') + } + return sha256(varPath, readFile) + } + const head = git(['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() + if (head !== config.fenceCommit) throw new Error('fence commit does not match checked-out HEAD') + const status = git(['status', '--porcelain=v1', '--untracked-files=no'], { + encoding: 'utf8' + }).trim() + if (status) throw new Error('Terraform fence requires a clean checkout') + const terraformStatus = git( + ['status', '--porcelain=v1', '--untracked-files=all', '--', config.terraformDir], + { encoding: 'utf8' } + ).trim() + if (terraformStatus) throw new Error('Terraform fence directory contains unreviewed files') + git(['ls-files', '--error-unmatch', '--', varPath], { encoding: 'utf8' }) + return sha256(varPath, readFile) +} + +function assertAttemptCheckoutBinding(config, attempt, deps) { + const digest = assertReviewedFenceCheckout(config, deps) + if (digest !== attempt.varFileSha256) { + throw new Error('reviewed Terraform variable-file digest changed') + } +} + +function assertReplayStateIdentity(progress, attempt) { + if ( + progress.stateLineage !== attempt.terraformStateLineage || + progress.stateSerial !== attempt.terraformStateSerial + ) { + throw new Error('Terraform state lineage or serial changed before saved-plan replay') + } +} + +function assertStateObjectBinding(binding, attempt) { + if ( + binding.generation !== attempt.terraformStateObjectGeneration || + binding.sha256 !== attempt.terraformStateObjectSha256 || + binding.lineage !== attempt.terraformStateLineage || + binding.serial !== attempt.terraformStateSerial + ) { + throw new Error('Terraform pre-state object generation or digest changed') + } +} + +function completedStateBranch(progress, attempt) { + if (progress.stateLineage !== attempt.terraformStateLineage) { + throw new Error('completed Terraform fence state identity is unexpected') + } + if (progress.stateSerial === attempt.terraformStateSerial) return 'replay' + if (progress.stateSerial === attempt.terraformStateSerial + 1) return 'complete' + throw new Error('completed Terraform fence state identity is unexpected') +} + +async function persistInvocationOperations(attempt, progress, markOperation) { + let updated = attempt + for (const observed of progress.invocationOperations ?? []) { + const recorded = (updated.applyInvocations ?? []).find( + (value) => value.invocationId === observed.invocationId + ) + if (!observed.gceOperation || recorded?.gceOperation) continue + const marked = await markOperation( + { ...updated, gceOperation: observed.gceOperation }, + observed + ) + updated = { + ...marked.attempt, + applyInvocations: (updated.applyInvocations ?? []).map((value) => + value.invocationId === marked.invocation.invocationId + ? marked.invocation + : value + ) + } + } + return updated +} + +export function assertTerraformFenceZeroDiff(config, deps = {}) { + const terraform = deps.terraform ?? defaultTerraform + const directory = privatePlanDirectory({ + mkdtemp: deps.mkdtemp ?? mkdtempSync, + chmod: deps.chmod ?? chmodSync, + tmpdir: deps.tmpdir ?? tmpdir + }) + const planPath = join(directory, 'completion.tfplan') + try { + terraform( + [ + `-chdir=${config.terraformDir}`, + 'plan', + '-input=false', + '-refresh=false', + `-lock-timeout=${config.lockTimeout}`, + `-var-file=${config.varFile}`, + `-target=${MIG_ADDRESS_PREFIX}["${config.cell.cellId}"]`, + `-out=${planPath}`, + '-no-color' + ], + { encoding: 'utf8' } + ) + const plan = terraformJson({ terraform }, [ + `-chdir=${config.terraformDir}`, + 'show', + '-json', + planPath + ]) + validateTerraformFenceCompletionPlan(plan, exactMigExpected(config.cell)) + } catch { + throw new Error('completed Terraform fence has an unsafe reviewed diff') + } finally { + ;(deps.remove ?? rmSync)(directory, { recursive: true, force: true }) + } +} + +export function assertTerraformFenceStateFenced(config, deps = {}) { + const terraform = deps.terraform ?? defaultTerraform + const state = terraformJson({ terraform }, [ + `-chdir=${config.terraformDir}`, + 'show', + '-json' + ]) + if (terraformFenceState(state, exactMigExpected(config.cell)) !== 0) { + throw new Error('Terraform state does not record the requested cell fence') + } +} + +export async function adoptLegacyTerraformFence(config, deps) { + for (const dependency of [ + 'loadAttempt', + 'assertCommittedFenceSet', + 'assertStateFenced', + 'preApplyGuard', + 'postApplyGuard', + 'attest', + 'commitAdoption' + ]) { + if (typeof deps[dependency] !== 'function') throw new Error(`missing ${dependency} dependency`) + } + assertFenceIdentity(config.cell) + assertReviewedFenceCheckout(config, deps) + if (await deps.loadAttempt()) { + throw new Error('legacy Terraform fence adoption requires no durable attempt') + } + await deps.assertCommittedFenceSet() + await deps.preApplyGuard() + await deps.assertStateFenced() + await deps.postApplyGuard(config.cellIncarnation) + if (await deps.loadAttempt()) { + throw new Error('durable fence attempt appeared during legacy adoption') + } + await deps.attest(config.cellIncarnation) + await deps.postApplyGuard(config.cellIncarnation) + await deps.commitAdoption(config.cellIncarnation) + deps.emit?.({ + event: 'terraform_cell_fence_legacy_adopted', + cellId: config.cell.cellId + }) +} + +function planObjectUri(config, attempt) { + return `gs://${config.project}-terraform-state/${attempt.planObjectName}` +} + +export async function readTerraformStateObjectBinding( + config, + deps, + statePath +) { + const uri = `gs://${config.project}-terraform-state/terraform/state/default.tfstate` + const metadata = deps.commandJson([ + 'storage', + 'objects', + 'describe', + uri, + '--format=json' + ]) + const generation = String(metadata.generation ?? '') + if (!/^[1-9][0-9]{0,30}$/.test(generation)) { + throw new Error('Terraform state object has no valid generation') + } + deps.command([ + 'storage', + 'cp', + `${uri}#${generation}`, + statePath, + `--if-generation-match=${generation}`, + '--quiet' + ]) + ;(deps.chmod ?? chmodSync)(statePath, 0o600) + const contents = (deps.readFile ?? readFileSync)(statePath) + const state = JSON.parse(contents.toString()) + if ( + !TERRAFORM_STATE_LINEAGE.test(state.lineage ?? '') || + !Number.isSafeInteger(state.serial) || + state.serial < 0 + ) { + throw new Error('Terraform state object identity is invalid') + } + return { + generation, + sha256: createHash('sha256').update(contents).digest('hex'), + lineage: state.lineage, + serial: state.serial + } +} + +export async function uploadTerraformFencePlan(config, deps, planPath, attempt) { + const uri = planObjectUri(config, attempt) + deps.command([ + 'storage', + 'cp', + planPath, + uri, + '--if-generation-match=0', + '--quiet' + ]) + const metadata = deps.commandJson([ + 'storage', + 'objects', + 'describe', + uri, + '--format=json' + ]) + const generation = String(metadata.generation ?? '') + if (!/^[1-9][0-9]{0,30}$/.test(generation)) { + throw new Error('uploaded fence plan has no valid object generation') + } + return { generation } +} + +export async function resolveTerraformFencePlanGeneration(config, deps, attempt) { + const result = deps.commandResult([ + 'storage', + 'objects', + 'describe', + planObjectUri(config, attempt), + '--format=value(generation)' + ]) + if (result.status !== 0) { + if (/(?:404|not found|no urls matched)/i.test(result.stderr ?? '')) { + return { generation: null } + } + throw new Error('durable fence plan object could not be inspected') + } + const generation = String(result.stdout ?? '').trim() + if (!/^[1-9][0-9]{0,30}$/.test(generation)) { + throw new Error('durable fence plan has no valid object generation') + } + return { generation } +} + +export async function downloadTerraformFencePlan(config, deps, attempt, planPath) { + const uri = `${planObjectUri(config, attempt)}#${attempt.planObjectGeneration}` + deps.command([ + 'storage', + 'cp', + uri, + planPath, + `--if-generation-match=${attempt.planObjectGeneration}`, + '--quiet' + ]) + ;(deps.chmod ?? chmodSync)(planPath, 0o600) +} + +export async function deleteTerraformFencePlan(config, deps, attempt) { + const result = deps.commandResult([ + 'storage', + 'rm', + `${planObjectUri(config, attempt)}#${attempt.planObjectGeneration}`, + `--if-generation-match=${attempt.planObjectGeneration}`, + '--quiet' + ]) + if (result.status === 0) return + if (/(?:404|not found|no urls matched)/i.test(result.stderr ?? '')) return + throw new Error('exact Terraform fence plan generation could not be deleted') +} + +export async function runTerraformFenceApply(config, overrides = {}) { + const terraform = overrides.terraform ?? defaultTerraform + const deps = { + terraform, + git: overrides.git ?? defaultGit, + mkdtemp: overrides.mkdtemp ?? mkdtempSync, + chmod: overrides.chmod ?? chmodSync, + tmpdir: overrides.tmpdir ?? tmpdir, + readFile: overrides.readFile ?? readFileSync, + remove: overrides.remove ?? rmSync, + stat: overrides.stat ?? statSync, + randomUUID: overrides.randomUUID ?? randomUUID, + inspectProgress: overrides.inspectProgress, + prepareAttempt: overrides.prepareAttempt, + bindPlan: overrides.bindPlan, + markApplyStarted: overrides.markApplyStarted, + markOperation: overrides.markOperation, + attest: overrides.attest, + uploadPlan: overrides.uploadPlan, + deletePlan: overrides.deletePlan, + stateObjectBinding: overrides.stateObjectBinding, + assertZeroDiff: + overrides.assertZeroDiff ?? + (async () => assertTerraformFenceZeroDiff(config, { terraform })), + preApplyGuard: overrides.preApplyGuard, + postApplyGuard: overrides.postApplyGuard, + assertCommittedFenceSet: + overrides.assertCommittedFenceSet ?? + (() => assertTerraformFenceSet(config, { terraform })), + emit: overrides.emit ?? (() => {}) + } + for (const dependency of [ + 'inspectProgress', + 'prepareAttempt', + 'bindPlan', + 'markApplyStarted', + 'markOperation', + 'attest', + 'uploadPlan', + 'deletePlan', + 'stateObjectBinding', + 'assertZeroDiff', + 'preApplyGuard', + 'postApplyGuard' + ]) { + if (typeof deps[dependency] !== 'function') throw new Error(`missing ${dependency} dependency`) + } + assertFenceIdentity(config.cell) + const varFileSha256 = assertReviewedFenceCheckout(config, deps) + await deps.assertCommittedFenceSet() + const expected = exactMigExpected(config.cell) + const directory = privatePlanDirectory(deps) + const planPath = join(directory, 'fence.tfplan') + try { + const initialProgress = await deps.inspectProgress(expected, null) + if (classifyTerraformFenceProgress(initialProgress) !== 'not-started') { + throw new Error('Terraform fence is not in the exact initial live state') + } + const stateBinding = await deps.stateObjectBinding( + join(directory, 'pre-state.tfstate') + ) + deps.terraform([ + `-chdir=${config.terraformDir}`, + 'plan', + '-input=false', + '-refresh=false', + `-lock-timeout=${config.lockTimeout}`, + `-var-file=${config.varFile}`, + `-target=${MIG_ADDRESS_PREFIX}["${config.cell.cellId}"]`, + `-out=${planPath}` + ]) + deps.chmod(planPath, 0o600) + const postPlanStateBinding = await deps.stateObjectBinding( + join(directory, 'post-plan-state.tfstate') + ) + if ( + JSON.stringify(postPlanStateBinding) !== JSON.stringify(stateBinding) + ) { + throw new Error('Terraform state object changed while creating the fence plan') + } + if ((deps.stat(planPath).mode & 0o077) !== 0) { + throw new Error('saved fence plan permissions are not private') + } + const plan = terraformJson(deps, [ + `-chdir=${config.terraformDir}`, + 'show', + '-json', + planPath + ]) + validateTerraformFencePlan(plan, expected) + const planSha256 = sha256(planPath, deps.readFile) + const attemptId = deps.randomUUID() + const planObjectName = + `${PLAN_OBJECT_PREFIX}/${config.environment}/${attemptId}.tfplan` + const attempt = { + attemptId, + environment: config.environment, + cellId: config.cell.cellId, + cellIncarnation: config.cellIncarnation, + migName: config.cell.migName, + instanceGroup: config.cell.instanceGroup, + generationIdentity: config.cell.generationIdentity, + fenceCommit: config.fenceCommit, + planSha256, + planObjectName, + varFileSha256, + terraformStateLineage: stateBinding.lineage, + terraformStateSerial: stateBinding.serial, + terraformStateObjectGeneration: stateBinding.generation, + terraformStateObjectSha256: stateBinding.sha256, + requestReason: `orca-relay-fence/${attemptId}` + } + const prepared = await deps.prepareAttempt(attempt) + let durableAttempt = prepared?.attempt ?? prepared + if ( + !durableAttempt || + !Number.isSafeInteger(durableAttempt.createdAt) || + !Number.isSafeInteger(durableAttempt.expiresAt) + ) { + throw new Error('durable fence attempt has no creation or expiry time') + } + assertAttemptMatches(config, durableAttempt, false) + const uploaded = await deps.uploadPlan(planPath, durableAttempt) + const bound = await deps.bindPlan({ + ...durableAttempt, + planObjectGeneration: uploaded.generation + }) + durableAttempt = bound?.attempt ?? bound + assertAttemptMatches(config, durableAttempt) + assertAttemptCheckoutBinding(config, durableAttempt, deps) + await deps.preApplyGuard() + validateTerraformFencePlan( + terraformJson(deps, [ + `-chdir=${config.terraformDir}`, + 'show', + '-json', + planPath + ]), + expected + ) + if (sha256(planPath, deps.readFile) !== planSha256) { + throw new Error('saved fence plan digest changed') + } + assertStateObjectBinding( + await deps.stateObjectBinding(join(directory, 'pre-apply-state.tfstate')), + durableAttempt + ) + const invocationId = deps.randomUUID() + const invocationRequestReason = + `${durableAttempt.requestReason}/${invocationId}` + const startedResult = await deps.markApplyStarted(durableAttempt, { + invocationId, + requestReason: invocationRequestReason + }) + const startedAttempt = { + ...startedResult.attempt, + applyInvocations: [ + ...(durableAttempt.applyInvocations ?? []), + startedResult.invocation + ] + } + if (!Number.isSafeInteger(startedAttempt?.applyStartedAt)) { + throw new Error('durable fence attempt has no apply-start time') + } + let applyError + try { + deps.terraform( + [ + `-chdir=${config.terraformDir}`, + 'apply', + '-input=false', + `-lock-timeout=${config.lockTimeout}`, + planPath + ], + { + env: { + ...process.env, + GOOGLE_REQUEST_REASON: invocationRequestReason + } + } + ) + } catch (error) { + applyError = error + } + const progress = await deps.inspectProgress(expected, startedAttempt) + const classification = classifyTerraformFenceProgress(progress) + const observedAttempt = await persistInvocationOperations( + startedAttempt, + progress, + deps.markOperation + ) + if (classification !== 'complete') { + const reason = applyError ? 'apply response failed' : 'apply did not converge' + throw new Error(`${reason}; recover-forward required`) + } + if (!progress.gceOperation) throw new Error('completed fence has no GCE operation evidence') + if (completedStateBranch(progress, startedAttempt) !== 'complete') { + throw new Error('Terraform state did not persist the completed fence') + } + await deps.assertZeroDiff() + await deps.postApplyGuard(startedAttempt.cellIncarnation) + const completedAttempt = { + ...observedAttempt, + gceOperation: progress.gceOperation + } + await deps.attest(completedAttempt) + await deps.deletePlan(completedAttempt) + deps.emit({ + event: 'terraform_cell_fenced', + cellId: config.cell.cellId, + attemptId: startedAttempt.attemptId, + planSha256 + }) + return durableAttempt + } finally { + deps.remove(directory, { recursive: true, force: true }) + } +} + +export async function inspectTerraformFenceProgress(config, deps, attempt) { + const terraform = deps.terraform ?? defaultTerraform + const expected = exactMigExpected(config.cell) + const stateIdentity = terraformFenceStateIdentity(config, { terraform }) + const state = terraformJson({ terraform }, [ + `-chdir=${config.terraformDir}`, + 'show', + '-json' + ]) + const stateTargetSize = terraformFenceState(state, expected) + const live = deps.gcloudJson([ + 'compute', + 'instance-groups', + 'managed', + 'describe', + config.cell.migName, + '--project', + config.project, + '--zone', + config.cell.zone, + '--format=json' + ]) + const instances = deps.gcloudJson([ + 'compute', + 'instance-groups', + 'managed', + 'list-instances', + config.cell.migName, + '--project', + config.project, + '--zone', + config.cell.zone, + '--format=json' + ]) + const operations = deps.gcloudJson([ + 'compute', + 'operations', + 'list', + '--project', + config.project, + `--filter=zone:(${config.cell.zone}) AND targetLink:${config.cell.migName}`, + '--sort-by=~insertTime', + '--limit=20', + '--format=json' + ]) + const operationCandidates = operations.filter((operation) => { + const insertedAt = Date.parse(operation.insertTime) + return ( + typeof operation.name === 'string' && + operation.targetLink === + `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` && + operation.operationType === 'compute.instanceGroupManagers.resize' && + Number.isSafeInteger(attempt?.applyStartedAt) && + Number.isFinite(insertedAt) && + insertedAt >= attempt.applyStartedAt + ) + }) + const invocations = attempt?.applyInvocations ?? [] + if (attempt?.applyStartedAt && invocations.length === 0) { + throw new Error('Terraform fence apply has no durable invocation ledger') + } + const auditEntries = invocations.length > 0 + ? deps.gcloudJson([ + 'logging', + 'read', + `protoPayload.requestMetadata.requestAttributes.reason:"${attempt.requestReason}/" AND protoPayload.resourceName="projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}"`, + '--project', + config.project, + '--limit=20', + '--format=json' + ]) + : [] + const invocationOperations = invocations.map((invocation) => { + const matchingAudits = (Array.isArray(auditEntries) ? auditEntries : []).filter((entry) => { + const payload = entry.protoPayload ?? {} + return ( + payload.requestMetadata?.requestAttributes?.reason === invocation.requestReason && + payload.resourceName === + `projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` && + String(payload.methodName ?? '').endsWith('instanceGroupManagers.resize') && + Number(payload.request?.size ?? payload.request?.targetSize) === 0 + ) + }) + if (matchingAudits.length > 1) { + throw new Error('Terraform fence invocation has ambiguous audit operations') + } + const responseName = String( + matchingAudits[0]?.protoPayload?.response?.name ?? '' + ) + const operationName = responseName.includes('/operations/') + ? responseName.slice(responseName.lastIndexOf('/') + 1) + : responseName + const expectedName = invocation.gceOperation ?? operationName + const operation = expectedName + ? operationCandidates.find((candidate) => candidate.name === expectedName) + : undefined + if ( + (invocation.gceOperation && operationName && invocation.gceOperation !== operationName) || + (expectedName && !operation) + ) { + throw new Error('Terraform fence invocation operation mismatch') + } + return { + ...invocation, + gceOperation: operation?.name, + operationStatus: operation?.status ?? 'ABSENT', + operationError: Boolean(operation?.error), + auditBound: Boolean(matchingAudits.length === 1 && operation) + } + }) + const boundOperations = invocationOperations.filter( + (invocation) => invocation.gceOperation + ) + const finalOperation = boundOperations.at(-1) + const operationStatus = invocationOperations.some((invocation) => + ['PENDING', 'RUNNING'].includes(invocation.operationStatus) + ) + ? 'RUNNING' + : (finalOperation?.operationStatus ?? 'ABSENT') + return { + stateTargetSize, + stateLineage: stateIdentity.lineage, + stateSerial: stateIdentity.serial, + liveTargetSize: Number(live.targetSize), + instanceCount: instances.length, + operationStatus, + operationError: invocationOperations.some( + (invocation) => invocation.operationError + ), + operationAuditBound: + boundOperations.length > 0 && + boundOperations.every((invocation) => invocation.auditBound), + gceOperation: finalOperation?.gceOperation, + invocationOperations + } +} + +function responseOperationName(entry) { + const name = String(entry?.protoPayload?.response?.name ?? '') + return name.includes('/operations/') + ? name.slice(name.lastIndexOf('/') + 1) + : name +} + +export async function inspectCompletedTerraformFenceProgress( + config, + deps, + attempt, + recovery +) { + if (!Number.isSafeInteger(attempt?.applyStartedAt)) { + throw new Error('completed fence recovery has no durable apply start') + } + const terraform = deps.terraform ?? defaultTerraform + const expected = exactMigExpected(config.cell) + const stateIdentity = terraformFenceStateIdentity(config, { terraform }) + const state = terraformJson({ terraform }, [ + `-chdir=${config.terraformDir}`, + 'show', + '-json' + ]) + const stateTargetSize = terraformFenceState(state, expected) + const live = deps.gcloudJson([ + 'compute', + 'instance-groups', + 'managed', + 'describe', + config.cell.migName, + '--project', + config.project, + '--zone', + config.cell.zone, + '--format=json' + ]) + const instances = deps.gcloudJson([ + 'compute', + 'instance-groups', + 'managed', + 'list-instances', + config.cell.migName, + '--project', + config.project, + '--zone', + config.cell.zone, + '--format=json' + ]) + const targetLink = + `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` + const operations = deps.gcloudJson([ + 'compute', + 'operations', + 'list', + '--project', + config.project, + `--filter=zone:(${config.cell.zone}) AND targetLink:${config.cell.migName}`, + '--sort-by=~insertTime', + '--limit=20', + '--format=json' + ]) + const operationCandidates = operations.filter((operation) => { + const insertedAt = Date.parse(operation.insertTime) + return ( + typeof operation.name === 'string' && + operation.targetLink === targetLink && + operation.operationType === 'compute.instanceGroupManagers.resize' && + Number.isFinite(insertedAt) && + insertedAt >= attempt.applyStartedAt + ) + }) + if ( + operationCandidates.length !== 1 || + operationCandidates[0].name !== recovery.gceOperation + ) { + throw new Error('completed fence recovery has no unique Compute operation') + } + const resourceName = + `projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` + const auditEntries = deps.gcloudJson([ + 'logging', + 'read', + `protoPayload.authenticationInfo.principalEmail="${recovery.principalEmail}" AND protoPayload.resourceName="${resourceName}" AND protoPayload.methodName:"instanceGroupManagers.resize" AND timestamp>="${new Date(attempt.applyStartedAt).toISOString()}"`, + '--project', + config.project, + '--limit=20', + '--format=json' + ]) + const matchingAudits = (Array.isArray(auditEntries) ? auditEntries : []).filter( + (entry) => { + const payload = entry.protoPayload ?? {} + const timestamp = Date.parse(entry.timestamp) + return ( + payload.authenticationInfo?.principalEmail === recovery.principalEmail && + payload.resourceName === resourceName && + String(payload.methodName ?? '').endsWith('instanceGroupManagers.resize') && + Number(payload.request?.size ?? payload.request?.targetSize) === 0 && + Number.isFinite(timestamp) && + timestamp >= attempt.applyStartedAt && + responseOperationName(entry) === recovery.gceOperation + ) + } + ) + if (matchingAudits.length !== 1) { + throw new Error('completed fence recovery has no unique Audit Log operation') + } + const [operation] = operationCandidates + return { + stateTargetSize, + stateLineage: stateIdentity.lineage, + stateSerial: stateIdentity.serial, + liveTargetSize: Number(live.targetSize), + instanceCount: instances.length, + liveStable: live.status?.isStable === true, + operationStatus: operation.status, + operationError: Boolean(operation.error), + gceOperation: operation.name + } +} + +export function assertTerraformFenceSet(config, deps = {}) { + const terraform = deps.terraform ?? defaultTerraform + const result = terraform( + [ + `-chdir=${config.terraformDir}`, + 'console', + `-var-file=${config.varFile}` + ], + { + encoding: 'utf8', + input: + `contains(var.relay_gce_fenced_cells, ${JSON.stringify(config.cell.cellId)}) && ` + + `try(local.relay_gce_cell_target_sizes[${JSON.stringify(config.cell.cellId)}], -1) == 0\n` + } + ) + if (result.trim() !== 'true') { + throw new Error('requested cell is not in the committed Terraform fence set') + } +} + +export async function recoverSupersededCompletedTerraformFence( + config, + deps, + recovery +) { + assertFenceIdentity(config.cell) + const varFileSha256 = assertReviewedFenceCheckout(config, deps) + await deps.assertCommittedFenceSet() + const attempt = await deps.loadAttempt(config.cell.cellId) + if ( + !attempt || + attempt.fenceCommit === config.fenceCommit || + attempt.attemptId !== recovery.attemptId || + attempt.fenceCommit !== recovery.fenceCommit || + attempt.terraformStateSerial !== recovery.terraformStateSerial || + attempt.planObjectGeneration !== recovery.planObjectGeneration + ) { + throw new Error('completed fence recovery does not match the pinned attempt') + } + assertAttemptMatches({ ...config, fenceCommit: attempt.fenceCommit }, attempt) + if ( + varFileSha256 !== attempt.varFileSha256 || + !Number.isSafeInteger(attempt.applyStartedAt) || + attempt.abortedAt || + (attempt.gceOperation && attempt.gceOperation !== recovery.gceOperation) + ) { + throw new Error('completed fence recovery attempt is not adoptable') + } + const invocations = attempt.applyInvocations ?? [] + if ( + invocations.length !== 1 || + (invocations[0].gceOperation && + invocations[0].gceOperation !== recovery.gceOperation) || + invocations[0].startedAt < attempt.applyStartedAt + ) { + throw new Error('completed fence recovery invocation ledger is unsafe') + } + const resolved = await deps.resolvePlan(attempt) + const planExists = resolved.generation === attempt.planObjectGeneration + if (!planExists && !(attempt.completedAt && resolved.generation === null)) { + throw new Error('completed fence recovery saved-plan generation changed') + } + const directory = privatePlanDirectory({ + mkdtemp: deps.mkdtemp ?? mkdtempSync, + chmod: deps.chmod ?? chmodSync, + tmpdir: deps.tmpdir ?? tmpdir + }) + try { + const stateBinding = await deps.stateObjectBinding( + join(directory, 'completed-state.tfstate') + ) + if ( + stateBinding.generation !== recovery.terraformStateObjectGeneration || + stateBinding.sha256 !== recovery.terraformStateObjectSha256 || + stateBinding.lineage !== attempt.terraformStateLineage || + stateBinding.serial !== attempt.terraformStateSerial + 1 + ) { + throw new Error('completed fence recovery state object changed') + } + if (planExists) { + const planPath = join(directory, 'fence.tfplan') + await deps.downloadPlan(attempt, planPath) + if ((deps.stat ?? statSync)(planPath).mode & 0o077) { + throw new Error('downloaded saved fence plan permissions are not private') + } + if (sha256(planPath, deps.readFile ?? readFileSync) !== attempt.planSha256) { + throw new Error('completed fence recovery saved-plan digest changed') + } + validateTerraformFencePlan( + terraformJson( + { terraform: deps.terraform ?? defaultTerraform }, + [`-chdir=${config.terraformDir}`, 'show', '-json', planPath] + ), + exactMigExpected(config.cell) + ) + } + const progress = await deps.inspectCompletedProgress( + exactMigExpected(config.cell), + attempt, + recovery + ) + if ( + progress.stateLineage !== attempt.terraformStateLineage || + progress.stateSerial !== attempt.terraformStateSerial + 1 || + progress.stateTargetSize !== 0 || + progress.liveTargetSize !== 0 || + progress.instanceCount !== 0 || + progress.liveStable !== true || + progress.operationStatus !== 'DONE' || + progress.operationError || + progress.gceOperation !== recovery.gceOperation + ) { + throw new Error('completed fence recovery production evidence is unsafe') + } + const invocation = { + ...invocations[0], + gceOperation: recovery.gceOperation + } + const marked = attempt.gceOperation + ? { attempt, invocation } + : await deps.markOperation( + { ...attempt, gceOperation: recovery.gceOperation }, + invocation + ) + await deps.assertZeroDiff() + await deps.postApplyGuard(attempt.cellIncarnation) + await deps.attest({ + ...marked.attempt, + applyInvocations: [marked.invocation], + gceOperation: recovery.gceOperation + }) + if (planExists) await deps.deletePlan(attempt) + deps.emit?.({ + event: 'terraform_cell_fence_completed_attempt_recovered', + cellId: config.cell.cellId, + attemptId: attempt.attemptId, + gceOperation: recovery.gceOperation + }) + } finally { + ;(deps.remove ?? rmSync)(directory, { recursive: true, force: true }) + } +} + +export async function resumeTerraformFence(config, deps) { + assertFenceIdentity(config.cell) + assertReviewedFenceCheckout(config, deps) + await deps.assertCommittedFenceSet() + let attempt = await deps.loadAttempt(config.cell.cellId) + assertAttemptMatches(config, attempt, false) + if (!attempt.planObjectGeneration) { + const resolved = await deps.resolvePlan(attempt) + if (!resolved.generation) throw new Error('durable fence plan object is missing') + const bound = await deps.bindPlan({ + ...attempt, + planObjectGeneration: resolved.generation + }) + attempt = bound?.attempt ?? bound + } + assertAttemptMatches(config, attempt) + assertAttemptCheckoutBinding(config, attempt, deps) + let progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) + let classification = classifyTerraformFenceProgress(progress) + if (classification === 'complete') { + if (completedStateBranch(progress, attempt) === 'replay') { + classification = 'reconcile-state' + } else { + await deps.assertZeroDiff() + } + } + if (classification !== 'complete') { + if (classification === 'in-progress' && Number.isSafeInteger(attempt.applyStartedAt)) { + throw new Error('Terraform fence operation is still running; recover-forward required') + } + attempt = await persistInvocationOperations( + attempt, + progress, + deps.markOperation + ) + assertReplayStateIdentity(progress, attempt) + await deps.preApplyGuard() + const directory = privatePlanDirectory({ + mkdtemp: deps.mkdtemp ?? mkdtempSync, + chmod: deps.chmod ?? chmodSync, + tmpdir: deps.tmpdir ?? tmpdir + }) + const planPath = join(directory, 'fence.tfplan') + try { + assertStateObjectBinding( + await deps.stateObjectBinding( + join(directory, 'replay-pre-state.tfstate') + ), + attempt + ) + await deps.downloadPlan(attempt, planPath) + const stat = (deps.stat ?? statSync)(planPath) + if ((stat.mode & 0o077) !== 0) { + throw new Error('downloaded saved fence plan permissions are not private') + } + if (sha256(planPath, deps.readFile ?? readFileSync) !== attempt.planSha256) { + throw new Error('downloaded saved fence plan digest mismatch') + } + validateTerraformFencePlan( + terraformJson( + { terraform: deps.terraform ?? defaultTerraform }, + [`-chdir=${config.terraformDir}`, 'show', '-json', planPath] + ), + exactMigExpected(config.cell) + ) + const invocationId = (deps.randomUUID ?? randomUUID)() + const invocationRequestReason = `${attempt.requestReason}/${invocationId}` + const started = await deps.markApplyStarted(attempt, { + invocationId, + requestReason: invocationRequestReason + }) + attempt = { + ...started.attempt, + applyInvocations: [ + ...(attempt.applyInvocations ?? []), + started.invocation + ] + } + let applyError + try { + ;(deps.terraform ?? defaultTerraform)( + [ + `-chdir=${config.terraformDir}`, + 'apply', + '-input=false', + `-lock-timeout=${config.lockTimeout}`, + planPath + ], + { + env: { + ...process.env, + GOOGLE_REQUEST_REASON: invocationRequestReason + } + } + ) + } catch (error) { + applyError = error + } + progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) + classification = classifyTerraformFenceProgress(progress) + attempt = await persistInvocationOperations( + attempt, + progress, + deps.markOperation + ) + if (classification !== 'complete') { + const reason = applyError ? 'saved-plan replay response failed' : 'saved-plan replay did not converge' + throw new Error(`${reason}; recover-forward required`) + } + if (completedStateBranch(progress, attempt) !== 'complete') { + throw new Error('Terraform state did not persist the replayed fence') + } + await deps.assertZeroDiff() + } finally { + ;(deps.remove ?? rmSync)(directory, { recursive: true, force: true }) + } + } + const gceOperation = progress.gceOperation ?? attempt.gceOperation + if (!gceOperation) throw new Error('completed fence has no GCE operation evidence') + await deps.postApplyGuard(attempt.cellIncarnation) + await deps.attest({ ...attempt, gceOperation }) + await deps.deletePlan(attempt) + deps.emit?.({ + event: 'terraform_cell_fence_resumed', + cellId: config.cell.cellId, + attemptId: attempt.attemptId + }) +} + +export async function abortTerraformFenceBeforeApply(config, deps) { + assertFenceIdentity(config.cell) + assertReviewedFenceCheckout(config, deps) + await deps.assertCommittedFenceSet() + let attempt = await deps.loadAttempt(config.cell.cellId) + assertAttemptMatches(config, attempt, false) + if (!attempt.planObjectGeneration) { + const resolved = await deps.resolvePlan(attempt) + if (resolved.generation) { + const bound = await deps.bindPlan({ + ...attempt, + planObjectGeneration: resolved.generation + }) + attempt = bound?.attempt ?? bound + } + } + assertAttemptMatches(config, attempt, Boolean(attempt.planObjectGeneration)) + assertAttemptCheckoutBinding(config, attempt, deps) + const progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) + if (classifyTerraformFenceProgress(progress) !== 'not-started') { + throw new Error('cannot abort after Terraform fence apply may have started') + } + if (attempt.applyStartedAt) throw new Error('cannot abort after Terraform fence apply was marked') + await deps.abortAttempt(attempt) + if (attempt.planObjectGeneration) await deps.deletePlan(attempt) + deps.emit?.({ event: 'terraform_fence_aborted_before_apply', cellId: config.cell.cellId }) +} + +export async function abortSupersededTerraformFenceBeforeUpload(config, deps) { + assertFenceIdentity(config.cell) + const varFileSha256 = assertReviewedFenceCheckout(config, deps) + await deps.assertCommittedFenceSet() + const attempt = await deps.loadAttempt(config.cell.cellId) + if ( + !attempt || + !/^[a-f0-9]{40}$/.test(attempt.fenceCommit ?? '') || + attempt.fenceCommit === config.fenceCommit + ) { + throw new Error('prepared Terraform fence attempt is not from a superseded commit') + } + assertAttemptMatches({ ...config, fenceCommit: attempt.fenceCommit }, attempt, false) + if (attempt.varFileSha256 !== varFileSha256) { + throw new Error('superseded Terraform fence variable-file digest changed') + } + if ( + attempt.planObjectGeneration || + attempt.applyStartedAt || + attempt.completedAt || + attempt.gceOperation || + (attempt.applyInvocations?.length ?? 0) > 0 + ) { + throw new Error('cannot supersede a Terraform fence attempt after plan upload') + } + const resolved = await deps.resolvePlan(attempt) + if (resolved?.generation !== null) { + throw new Error('cannot supersede a Terraform fence attempt with a saved plan') + } + const progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) + if (classifyTerraformFenceProgress(progress) !== 'not-started') { + throw new Error('cannot supersede after Terraform fence apply may have started') + } + await deps.abortAttempt(attempt) + deps.emit?.({ + event: 'terraform_fence_superseded_before_upload', + cellId: config.cell.cellId, + previousFenceCommit: attempt.fenceCommit, + fenceCommit: config.fenceCommit + }) +} diff --git a/cloud/dev/scripts/relay-gce-terraform-fence.test.mjs b/cloud/dev/scripts/relay-gce-terraform-fence.test.mjs new file mode 100644 index 00000000000..306e28c887b --- /dev/null +++ b/cloud/dev/scripts/relay-gce-terraform-fence.test.mjs @@ -0,0 +1,1522 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { + abortSupersededTerraformFenceBeforeUpload, + abortTerraformFenceBeforeApply, + adoptLegacyTerraformFence, + assertReviewedFenceCheckout, + assertTerraformFenceSet, + assertTerraformFenceStateFenced, + assertTerraformFenceZeroDiff, + classifyTerraformFenceProgress, + deleteTerraformFencePlan, + inspectCompletedTerraformFenceProgress, + inspectTerraformFenceProgress, + recoverSupersededCompletedTerraformFence, + runTerraformFenceApply, + resumeTerraformFence, + terraformFenceState, + terraformProcessStdio, + validateTerraformFenceCompletionPlan, + validateTerraformFencePlan +} from './relay-gce-terraform-fence.mjs' + +const cell = { + cellId: 'production-gce-c1', + migName: 'orca-relay-c1', + zone: 'us-central1-a', + instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', + generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', + fenced: true, + desiredTargetSize: 0 +} + +const expected = { + cellId: cell.cellId, + name: cell.migName, + zone: cell.zone, + instance_group: cell.instanceGroup, + generationIdentity: cell.generationIdentity +} +const stateLineage = 'c739dab4-e6e1-e627-02a9-504b3dda1a2c' + +test('adopts a legacy fence only after repeated no-op and live guards', async () => { + const events = [] + const calls = [] + await adoptLegacyTerraformFence( + { + environment: 'production', + terraformDir: 'infra/terraform', + varFile: 'environments/production.tfvars', + fenceCommit: 'a'.repeat(40), + cellIncarnation: '11111111-1111-4111-8111-111111111111', + cell + }, + { + environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40) }, + readFile: () => Buffer.from('reviewed variables'), + loadAttempt: async () => { + calls.push('attempt') + return null + }, + assertCommittedFenceSet: async () => calls.push('fence-set'), + preApplyGuard: async () => calls.push('pre'), + assertStateFenced: async () => calls.push('state-fenced'), + postApplyGuard: async () => calls.push('post'), + attest: async () => calls.push('attest'), + commitAdoption: async () => calls.push('commit'), + emit: (event) => events.push(event) + } + ) + assert.deepEqual(calls, [ + 'attempt', + 'fence-set', + 'pre', + 'state-fenced', + 'post', + 'attempt', + 'attest', + 'post', + 'commit' + ]) + assert.deepEqual(events, [ + { event: 'terraform_cell_fence_legacy_adopted', cellId: cell.cellId } + ]) +}) + +test('refuses legacy adoption when a durable attempt appears during proof', async () => { + let reads = 0 + let attested = false + await assert.rejects( + adoptLegacyTerraformFence( + { + environment: 'production', + terraformDir: 'infra/terraform', + varFile: 'environments/production.tfvars', + fenceCommit: 'a'.repeat(40), + cellIncarnation: '11111111-1111-4111-8111-111111111111', + cell + }, + { + environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40) }, + readFile: () => Buffer.from('reviewed variables'), + loadAttempt: async () => (++reads === 1 ? null : { attemptId: 'new' }), + assertCommittedFenceSet: async () => {}, + preApplyGuard: async () => {}, + assertStateFenced: async () => {}, + postApplyGuard: async () => {}, + attest: async () => { + attested = true + }, + commitAdoption: async () => {} + } + ), + /durable fence attempt appeared/ + ) + assert.equal(attested, false) +}) + +test('does not commit legacy adoption when the final guard fails', async () => { + let postGuards = 0 + let committed = false + await assert.rejects( + adoptLegacyTerraformFence( + { + environment: 'production', + terraformDir: 'infra/terraform', + varFile: 'environments/production.tfvars', + fenceCommit: 'a'.repeat(40), + cellIncarnation: '11111111-1111-4111-8111-111111111111', + cell + }, + { + environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40) }, + readFile: () => Buffer.from('reviewed variables'), + loadAttempt: async () => null, + assertCommittedFenceSet: async () => {}, + preApplyGuard: async () => {}, + assertStateFenced: async () => {}, + postApplyGuard: async () => { + postGuards++ + if (postGuards === 2) throw new Error('final guard failed') + }, + attest: async () => {}, + commitAdoption: async () => { + committed = true + } + } + ), + /final guard failed/ + ) + assert.equal(committed, false) +}) + +test('pipes reviewed Terraform console expressions to stdin', () => { + assert.deepEqual( + terraformProcessStdio({ encoding: 'utf8', input: 'contains(...)\n' }), + ['pipe', 'pipe', 'pipe'] + ) + assert.deepEqual(terraformProcessStdio({ encoding: 'utf8' }), [ + 'ignore', + 'pipe', + 'pipe' + ]) + assert.equal(terraformProcessStdio(), 'inherit') +}) + +test('checks the exact committed fence cell through Terraform console', () => { + let invocation + assertTerraformFenceSet( + { + terraformDir: 'infra/terraform', + varFile: 'environments/production.tfvars', + cell + }, + { + terraform: (args, options) => { + invocation = { args, options } + return 'true\n' + } + } + ) + assert.deepEqual(invocation.args, [ + '-chdir=infra/terraform', + 'console', + '-var-file=environments/production.tfvars' + ]) + assert.equal( + invocation.options.input, + 'contains(var.relay_gce_fenced_cells, "production-gce-c1") && ' + + 'try(local.relay_gce_cell_target_sizes["production-gce-c1"], -1) == 0\n' + ) +}) + +test('binds a gitless broker checkout to its immutable image commit', () => { + const config = { + fenceCommit: 'a'.repeat(40), + terraformDir: 'infra/terraform', + varFile: 'environments/production.tfvars' + } + const contents = Buffer.from('reviewed production variables') + const digest = assertReviewedFenceCheckout(config, { + environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: config.fenceCommit }, + readFile: () => contents, + git: () => { + throw new Error('git must not run inside the immutable broker image') + } + }) + assert.equal(digest, createHash('sha256').update(contents).digest('hex')) +}) + +test('rejects a broker image built for a different fence commit', () => { + assert.throws( + () => + assertReviewedFenceCheckout( + { + fenceCommit: 'a'.repeat(40), + terraformDir: 'infra/terraform', + varFile: 'environments/production.tfvars' + }, + { + environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'b'.repeat(40) }, + readFile: () => Buffer.from('reviewed production variables') + } + ), + /immutable broker image/ + ) +}) + +function plan(actions = ['update'], address = undefined) { + return { + resource_changes: [ + { + address: + address ?? + `google_compute_instance_group_manager.relay_gce_cell["${cell.cellId}"]`, + change: { + actions, + before: { + name: cell.migName, + zone: cell.zone, + instance_group: cell.instanceGroup, + target_size: 1, + version: [{ instance_template: cell.generationIdentity }] + }, + after: { + name: cell.migName, + zone: cell.zone, + instance_group: cell.instanceGroup, + target_size: 0, + version: [{ instance_template: cell.generationIdentity }] + } + } + } + ] + } +} + +function state(targetSize = 0) { + return { + values: { + root_module: { + resources: [ + { + address: + `google_compute_instance_group_manager.relay_gce_cell["${cell.cellId}"]`, + values: { + name: cell.migName, + zone: cell.zone, + instance_group: cell.instanceGroup, + target_size: targetSize, + version: [{ instance_template: cell.generationIdentity }] + } + } + ] + } + } + } +} + +test('accepts only one exact in-place MIG resize from one to zero', () => { + assert.equal(validateTerraformFencePlan(plan(), expected).change.after.target_size, 0) + for (const actions of [['create'], ['delete'], ['delete', 'create']]) { + assert.throws(() => validateTerraformFencePlan(plan(actions), expected)) + } + assert.throws(() => + validateTerraformFencePlan( + plan(['update'], 'google_compute_backend_service.relay_gce_cell["production-gce-c1"]'), + expected + ) + ) + const unrelated = plan() + unrelated.resource_changes.push({ + address: 'google_compute_url_map.relay_gce[0]', + change: { actions: ['update'], before: {}, after: {} } + }) + assert.throws(() => validateTerraformFencePlan(unrelated, expected)) +}) + +function completionPlan() { + const result = plan() + const before = result.resource_changes[0].change.before + const after = result.resource_changes[0].change.after + before.target_size = 0 + before.version[0].name = '0/2026-07-31 03:52:56.639922+00:00' + after.version[0].name = 'primary' + return result +} + +test('accepts only the empty MIG provider version-label normalization', () => { + assert.equal(validateTerraformFenceCompletionPlan({ resource_changes: [] }, expected), undefined) + assert.equal( + validateTerraformFenceCompletionPlan(completionPlan(), expected).change.after.version[0] + .name, + 'primary' + ) + + const resized = completionPlan() + resized.resource_changes[0].change.after.target_size = 1 + assert.throws(() => validateTerraformFenceCompletionPlan(resized, expected)) + + const replaced = completionPlan() + replaced.resource_changes[0].change.after.version[0].instance_template += '-other' + assert.throws(() => validateTerraformFenceCompletionPlan(replaced, expected)) + + const extraChange = completionPlan() + extraChange.resource_changes[0].change.after.update_policy = { type: 'PROACTIVE' } + assert.throws(() => validateTerraformFenceCompletionPlan(extraChange, expected)) + + const arbitraryLabel = completionPlan() + arbitraryLabel.resource_changes[0].change.before.version[0].name = 'other' + assert.throws(() => validateTerraformFenceCompletionPlan(arbitraryLabel, expected)) +}) + +test('binds Terraform state to the exact MIG generation', () => { + assert.equal(terraformFenceState(state(0), expected), 0) + const replaced = state(0) + replaced.values.root_module.resources[0].values.version[0].instance_template += '-other' + assert.throws(() => terraformFenceState(replaced, expected)) +}) + +test('requires Terraform state to record the exact completed fence', () => { + let invocation + assertTerraformFenceStateFenced(applyConfig(), { + terraform: (args) => { + invocation = args + return JSON.stringify(state(0)) + } + }) + assert.deepEqual(invocation, ['-chdir=infra/terraform', 'show', '-json']) + + assert.throws( + () => + assertTerraformFenceStateFenced(applyConfig(), { + terraform: () => JSON.stringify(state(1)) + }), + /does not record the requested cell fence/ + ) + const replaced = state(0) + replaced.values.root_module.resources[0].values.version[0].instance_template += '-other' + assert.throws(() => + assertTerraformFenceStateFenced(applyConfig(), { + terraform: () => JSON.stringify(replaced) + }) + ) +}) + +test('builds and verifies fence plans without unrelated live refreshes', () => { + let zeroDiffArgs + assertTerraformFenceZeroDiff(applyConfig(), { + terraform: (args) => { + if (args.includes('plan')) zeroDiffArgs = args + if (args.includes('show')) return JSON.stringify({ resource_changes: [] }) + } + }) + assert.equal(zeroDiffArgs.includes('-refresh=false'), true) + assert.equal( + zeroDiffArgs.includes( + '-target=google_compute_instance_group_manager.relay_gce_cell["production-gce-c1"]' + ), + true + ) +}) + +test('classifies complete, in-progress, and conclusively not-started fences', () => { + assert.equal( + classifyTerraformFenceProgress({ + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationAuditBound: true + }), + 'complete' + ) + assert.equal( + classifyTerraformFenceProgress({ + stateTargetSize: 1, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'RUNNING' + }), + 'in-progress' + ) + assert.equal( + classifyTerraformFenceProgress({ + stateTargetSize: 1, + liveTargetSize: 1, + instanceCount: 1, + operationStatus: 'ABSENT' + }), + 'not-started' + ) + assert.equal( + classifyTerraformFenceProgress({ + stateTargetSize: 1, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationAuditBound: true + }), + 'reconcile-state' + ) + assert.throws( + () => + classifyTerraformFenceProgress({ + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationAuditBound: false + }), + /audit binding/ + ) +}) + +function applyHarness({ + loseApplyResponse = false, + guardError = null, + postGuardError = null, + tamperPlan = false, + progress +} = {}) { + const root = mkdtempSync(join(tmpdir(), 'relay-fence-test-')) + const calls = [] + const applyEnvironments = [] + const events = [] + let planPath + let progressReads = 0 + const terraform = (args, options = {}) => { + calls.push(args) + if (args.includes('state') && args.includes('pull')) { + return JSON.stringify({ lineage: stateLineage, serial: 7 }) + } + if (args.includes('plan')) { + planPath = args.find((arg) => arg.startsWith('-out=')).slice(5) + writeFileSync(planPath, 'private saved plan', { mode: 0o644 }) + chmodSync(planPath, 0o644) + return + } + if (args.includes('show')) return JSON.stringify(plan()) + if (args.includes('apply')) { + applyEnvironments.push(options.env) + if (loseApplyResponse) throw new Error('lost response') + } + } + const evidence = [] + return { + root, + calls, + events, + evidence, + applyEnvironments, + planPath: () => planPath, + overrides: { + terraform, + git: (args) => { + if (args.includes('rev-parse')) return `${applyConfig().fenceCommit}\n` + return '' + }, + assertCommittedFenceSet: async () => {}, + tmpdir: () => root, + randomUUID: () => '11111111-1111-4111-8111-111111111111', + inspectProgress: async () => { + if (progressReads++ === 0) { + return { + stateTargetSize: 1, + liveTargetSize: 1, + instanceCount: 1, + operationStatus: 'ABSENT', + operationError: false, + operationAuditBound: false, + stateLineage, + stateSerial: 7, + invocationOperations: [] + } + } + return progress ?? { + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationError: false, + operationAuditBound: true, + stateLineage, + stateSerial: 8, + gceOperation: 'operation-1', + invocationOperations: [ + { + invocationId: '11111111-1111-4111-8111-111111111111', + requestReason: + 'orca-relay-fence/11111111-1111-4111-8111-111111111111/11111111-1111-4111-8111-111111111111', + startedAt: 101, + gceOperation: 'operation-1', + operationStatus: 'DONE', + operationError: false, + auditBound: true + } + ] + } + }, + uploadPlan: async () => ({ generation: '123456789' }), + stateObjectBinding: async () => ({ + generation: '987654321', + sha256: createHash('sha256').update('pre-state object').digest('hex'), + lineage: stateLineage, + serial: 7 + }), + bindPlan: async (value) => { + evidence.push(['bind', value]) + return { attempt: value } + }, + deletePlan: async (value) => evidence.push(['delete', value]), + assertZeroDiff: async () => {}, + prepareAttempt: async (value) => { + evidence.push(['prepare', value]) + return { attempt: { ...value, createdAt: 100, expiresAt: 3_600_100 } } + }, + markApplyStarted: async (value, invocation) => { + const started = { ...value, applyStartedAt: 101 } + const durableInvocation = { ...invocation, startedAt: 101 } + evidence.push(['started', started]) + return { attempt: started, invocation: durableInvocation } + }, + markOperation: async (value, invocation) => { + const durableInvocation = { + ...invocation, + gceOperation: value.gceOperation + } + evidence.push(['operation', value]) + return { attempt: value, invocation: durableInvocation } + }, + attest: async (value) => evidence.push(['attest', value]), + preApplyGuard: async () => { + if (guardError) throw guardError + if (tamperPlan) writeFileSync(planPath, 'tampered plan', { mode: 0o600 }) + }, + postApplyGuard: async () => { + if (postGuardError) throw postGuardError + }, + emit: (event) => events.push(event) + }, + cleanup: () => rmSync(root, { recursive: true, force: true }) + } +} + +function applyConfig() { + return { + project: 'project', + environment: 'production', + terraformDir: 'infra/terraform', + varFile: 'environments/production.tfvars', + lockTimeout: '5m', + fenceCommit: 'a'.repeat(40), + cellIncarnation: '22222222-2222-4222-8222-222222222222', + cell + } +} + +function durableAttempt(config = applyConfig(), overrides = {}) { + const attemptId = '11111111-1111-4111-8111-111111111111' + const varFile = join(config.terraformDir, config.varFile) + const result = { + attemptId, + environment: config.environment, + cellId: cell.cellId, + cellIncarnation: config.cellIncarnation, + migName: cell.migName, + instanceGroup: cell.instanceGroup, + generationIdentity: cell.generationIdentity, + fenceCommit: config.fenceCommit, + planSha256: createHash('sha256').update('private saved plan').digest('hex'), + planObjectName: `terraform/state/relay-fence-plans/${config.environment}/${attemptId}.tfplan`, + planObjectGeneration: '123456789', + varFileSha256: createHash('sha256').update(readFileSync(varFile)).digest('hex'), + terraformStateLineage: stateLineage, + terraformStateSerial: 7, + terraformStateObjectGeneration: '987654321', + terraformStateObjectSha256: createHash('sha256') + .update('pre-state object') + .digest('hex'), + requestReason: `orca-relay-fence/${attemptId}`, + createdAt: 100, + expiresAt: 3_600_100, + ...overrides + } + if (result.applyStartedAt && result.applyInvocations === undefined) { + const invocationId = '66666666-6666-4666-8666-666666666666' + result.applyInvocations = [ + { + invocationId, + requestReason: `${result.requestReason}/${invocationId}`, + startedAt: result.applyStartedAt, + gceOperation: result.gceOperation + } + ] + } + return result +} + +test('applies and attests the exact private saved plan', async () => { + const harness = applyHarness() + try { + await runTerraformFenceApply(applyConfig(), harness.overrides) + assert.deepEqual( + harness.evidence.map(([event]) => event), + ['prepare', 'bind', 'started', 'operation', 'attest', 'delete'] + ) + assert.equal(harness.calls.filter((args) => args.includes('apply')).length, 1) + const planArgs = harness.calls.find((args) => args.includes('plan')) + assert.equal(planArgs.includes('-refresh=false'), true) + assert.equal( + harness.applyEnvironments[0].GOOGLE_REQUEST_REASON, + 'orca-relay-fence/11111111-1111-4111-8111-111111111111/11111111-1111-4111-8111-111111111111' + ) + assert.equal(harness.events[0].event, 'terraform_cell_fenced') + assert.equal(existsSync(harness.planPath()), false) + } finally { + harness.cleanup() + } +}) + +test('rejects a malformed Terraform lineage before plan upload', async () => { + const harness = applyHarness() + const prepareAttempt = harness.overrides.prepareAttempt + harness.overrides.prepareAttempt = async (value) => { + const prepared = await prepareAttempt(value) + return { + attempt: { + ...prepared.attempt, + terraformStateLineage: 'not-a-terraform-lineage' + } + } + } + try { + await assert.rejects( + runTerraformFenceApply(applyConfig(), harness.overrides), + /valid Terraform state identity/ + ) + assert.deepEqual( + harness.evidence.map(([event]) => event), + ['prepare'] + ) + assert.equal(harness.calls.filter((args) => args.includes('apply')).length, 0) + } finally { + harness.cleanup() + } +}) + +test('recovers a successful fence after losing the apply response', async () => { + const harness = applyHarness({ loseApplyResponse: true }) + try { + await runTerraformFenceApply(applyConfig(), harness.overrides) + assert.equal(harness.evidence.some(([event]) => event === 'attest'), true) + } finally { + harness.cleanup() + } +}) + +test('does not start apply after a final pre-apply guard failure', async () => { + const harness = applyHarness({ guardError: new Error('guard failed') }) + try { + await assert.rejects( + runTerraformFenceApply(applyConfig(), harness.overrides), + /guard failed/ + ) + assert.equal(harness.calls.some((args) => args.includes('apply')), false) + assert.deepEqual(harness.evidence.map(([event]) => event), ['prepare', 'bind']) + } finally { + harness.cleanup() + } +}) + +test('rejects a saved plan whose digest changes before apply', async () => { + const harness = applyHarness({ tamperPlan: true }) + try { + await assert.rejects( + runTerraformFenceApply(applyConfig(), harness.overrides), + /digest changed/ + ) + assert.equal(harness.calls.some((args) => args.includes('apply')), false) + } finally { + harness.cleanup() + } +}) + +test('requires recover-forward when an apply remains in progress', async () => { + const harness = applyHarness({ + loseApplyResponse: true, + progress: { + stateTargetSize: 1, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'RUNNING', + operationError: false, + stateLineage, + stateSerial: 7, + gceOperation: 'operation-1' + } + }) + try { + await assert.rejects( + runTerraformFenceApply(applyConfig(), harness.overrides), + /recover-forward required/ + ) + assert.notEqual(harness.evidence.at(-1)[0], 'attest') + } finally { + harness.cleanup() + } +}) + +test('does not attest before retained topology and heartbeat guards pass', async () => { + const harness = applyHarness({ postGuardError: new Error('topology mismatch') }) + try { + await assert.rejects( + runTerraformFenceApply(applyConfig(), harness.overrides), + /topology mismatch/ + ) + assert.notEqual(harness.evidence.at(-1)[0], 'attest') + } finally { + harness.cleanup() + } +}) + +test('aborts only when state and live GCE prove apply never began', async () => { + let aborted = false + let deleted = false + const config = applyConfig() + const attempt = durableAttempt(config) + const common = { + git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), + loadAttempt: async () => attempt, + deletePlan: async () => { + deleted = true + } + } + await abortTerraformFenceBeforeApply(config, { + ...common, + assertCommittedFenceSet: async () => {}, + inspectProgress: async () => ({ + stateTargetSize: 1, + liveTargetSize: 1, + instanceCount: 1, + operationStatus: 'ABSENT', + stateLineage, + stateSerial: 7 + }), + abortAttempt: async () => { + aborted = true + } + }) + assert.equal(aborted, true) + assert.equal(deleted, true) + await assert.rejects( + abortTerraformFenceBeforeApply(config, { + ...common, + assertCommittedFenceSet: async () => {}, + inspectProgress: async () => ({ + stateTargetSize: 1, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'RUNNING', + stateLineage, + stateSerial: 7 + }), + abortAttempt: async () => {} + }), + /cannot abort/ + ) +}) + +test('supersedes only an older unuploaded fence attempt proven not started', async () => { + const config = applyConfig() + const attempt = durableAttempt(config, { + fenceCommit: 'b'.repeat(40), + planObjectGeneration: undefined + }) + const events = [] + let aborted = false + const common = { + git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), + assertCommittedFenceSet: async () => {}, + loadAttempt: async () => attempt, + resolvePlan: async () => ({ generation: null }), + inspectProgress: async () => ({ + stateTargetSize: 1, + liveTargetSize: 1, + instanceCount: 1, + operationStatus: 'ABSENT', + stateLineage, + stateSerial: 7 + }), + abortAttempt: async () => { + aborted = true + }, + emit: (event) => events.push(event) + } + await abortSupersededTerraformFenceBeforeUpload(config, common) + assert.equal(aborted, true) + assert.deepEqual(events, [ + { + event: 'terraform_fence_superseded_before_upload', + cellId: cell.cellId, + previousFenceCommit: 'b'.repeat(40), + fenceCommit: config.fenceCommit + } + ]) + + await assert.rejects( + abortSupersededTerraformFenceBeforeUpload(config, { + ...common, + loadAttempt: async () => ({ ...attempt, planObjectGeneration: '123456789' }) + }), + /after plan upload/ + ) + await assert.rejects( + abortSupersededTerraformFenceBeforeUpload(config, { + ...common, + resolvePlan: async () => ({ generation: '123456789' }) + }), + /with a saved plan/ + ) +}) + +test('resumes and attests when state and live GCE are already zero', async () => { + let attested + const config = applyConfig() + const attempt = durableAttempt(config, { + applyStartedAt: 101, + gceOperation: 'operation-1', + completedAt: 120 + }) + let deleted = false + await resumeTerraformFence(config, { + git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), + assertCommittedFenceSet: async () => {}, + loadAttempt: async () => attempt, + inspectProgress: async () => ({ + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationError: false, + operationAuditBound: true, + stateLineage, + stateSerial: 8, + gceOperation: 'operation-1' + }), + preApplyGuard: async () => {}, + postApplyGuard: async () => {}, + assertZeroDiff: async () => {}, + deletePlan: async () => { + deleted = true + }, + attest: async (value) => { + attested = value + } + }) + assert.equal(attested.attemptId, attempt.attemptId) + assert.equal(deleted, true) +}) + +function replayHarness({ + initialProgress, + finalProgress, + applyError = null, + zeroDiffError = null, + downloadedPlan = 'private saved plan', + attemptOverrides = {} +} = {}) { + const config = applyConfig() + const root = mkdtempSync(join(tmpdir(), 'relay-fence-replay-test-')) + const attempt = durableAttempt(config, { + applyStartedAt: 101, + ...attemptOverrides + }) + const progress = [ + initialProgress ?? { + stateTargetSize: 1, + liveTargetSize: 1, + instanceCount: 1, + operationStatus: 'ABSENT', + operationError: false, + stateLineage, + stateSerial: 7 + }, + finalProgress ?? { + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationError: false, + operationAuditBound: true, + stateLineage, + stateSerial: 8, + gceOperation: 'operation-1' + } + ] + let progressIndex = 0 + let applies = 0 + let downloads = 0 + let deleted = false + let attested = false + let zeroDiffChecks = 0 + return { + config, + attempt, + applies: () => applies, + downloads: () => downloads, + deleted: () => deleted, + attested: () => attested, + zeroDiffChecks: () => zeroDiffChecks, + deps: { + git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), + tmpdir: () => root, + assertCommittedFenceSet: async () => {}, + loadAttempt: async () => attempt, + resolvePlan: async () => ({ generation: '123456789' }), + bindPlan: async (value) => ({ attempt: value }), + inspectProgress: async (_expected, currentAttempt) => { + const value = progress[Math.min(progressIndex++, progress.length - 1)] + if (!value.gceOperation || value.invocationOperations) return value + const invocation = currentAttempt.applyInvocations?.at(-1) + return { + ...value, + invocationOperations: invocation + ? [ + { + ...invocation, + gceOperation: value.gceOperation, + operationStatus: value.operationStatus, + operationError: value.operationError, + auditBound: value.operationAuditBound + } + ] + : [] + } + }, + preApplyGuard: async () => {}, + postApplyGuard: async () => {}, + stateObjectBinding: async () => ({ + generation: attempt.terraformStateObjectGeneration, + sha256: attempt.terraformStateObjectSha256, + lineage: stateLineage, + serial: 7 + }), + assertZeroDiff: async () => { + zeroDiffChecks++ + if (zeroDiffError) throw zeroDiffError + }, + downloadPlan: async (value, path) => { + assert.equal( + value.planObjectGeneration, + attempt.planObjectGeneration ?? '123456789' + ) + downloads++ + writeFileSync(path, downloadedPlan, { mode: 0o600 }) + }, + terraform: (args) => { + if (args.includes('show')) return JSON.stringify(plan()) + if (args.includes('apply')) { + applies++ + if (applyError) throw applyError + } + }, + markOperation: async (value, invocation) => ({ + attempt: value, + invocation: { ...invocation, gceOperation: value.gceOperation } + }), + markApplyStarted: async (value, invocation) => ({ + attempt: { ...value, applyStartedAt: value.applyStartedAt ?? 101 }, + invocation: { ...invocation, startedAt: 101 } + }), + attest: async () => { + attested = true + }, + deletePlan: async () => { + deleted = true + } + }, + cleanup: () => rmSync(root, { recursive: true, force: true }) + } +} + +test('replays the exact durable plan after crashing immediately after apply-start', async () => { + const harness = replayHarness() + try { + await resumeTerraformFence(harness.config, harness.deps) + assert.equal(harness.downloads(), 1) + assert.equal(harness.applies(), 1) + assert.equal(harness.attested(), true) + assert.equal(harness.deleted(), true) + } finally { + harness.cleanup() + } +}) + +test('recovers an uploaded plan whose generation was not bound before runner loss', async () => { + const harness = replayHarness({ + attemptOverrides: { + planObjectGeneration: undefined, + applyStartedAt: undefined + } + }) + try { + await resumeTerraformFence(harness.config, harness.deps) + assert.equal(harness.downloads(), 1) + assert.equal(harness.applies(), 1) + assert.equal(harness.attested(), true) + } finally { + harness.cleanup() + } +}) + +test('replays the exact durable plan to reconcile live zero with stale state', async () => { + const harness = replayHarness({ + initialProgress: { + stateTargetSize: 1, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationError: false, + operationAuditBound: true, + stateLineage, + stateSerial: 7, + gceOperation: 'operation-1' + } + }) + try { + await resumeTerraformFence(harness.config, harness.deps) + assert.equal(harness.applies(), 1) + assert.equal(harness.attested(), true) + } finally { + harness.cleanup() + } +}) + +test('does not replay a stale saved plan after the first apply already persisted state', async () => { + const harness = replayHarness({ + initialProgress: { + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationError: false, + operationAuditBound: true, + stateLineage, + stateSerial: 8, + gceOperation: 'operation-1' + } + }) + try { + await resumeTerraformFence(harness.config, harness.deps) + assert.equal(harness.downloads(), 0) + assert.equal(harness.applies(), 0) + assert.equal(harness.deleted(), true) + assert.equal(harness.zeroDiffChecks(), 1) + } finally { + harness.cleanup() + } +}) + +test('replays when the recorded Terraform serial is unchanged even if refresh sees zero', async () => { + const harness = replayHarness({ + initialProgress: { + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationError: false, + operationAuditBound: true, + stateLineage, + stateSerial: 7, + gceOperation: 'operation-1' + } + }) + try { + await resumeTerraformFence(harness.config, harness.deps) + assert.equal(harness.applies(), 1) + assert.equal(harness.zeroDiffChecks(), 1) + } finally { + harness.cleanup() + } +}) + +test('freezes a serial-plus-one completion unless the reviewed targeted plan is zero diff', async () => { + const harness = replayHarness({ + initialProgress: { + stateTargetSize: 0, + liveTargetSize: 0, + instanceCount: 0, + operationStatus: 'DONE', + operationError: false, + operationAuditBound: true, + stateLineage, + stateSerial: 8, + gceOperation: 'operation-1' + }, + zeroDiffError: new Error('not zero diff') + }) + try { + await assert.rejects( + resumeTerraformFence(harness.config, harness.deps), + /not zero diff/ + ) + assert.equal(harness.attested(), false) + assert.equal(harness.deleted(), false) + } finally { + harness.cleanup() + } +}) + +test('rejects saved-plan object generation and hash mismatches', async () => { + const generation = replayHarness({ + attemptOverrides: { planObjectGeneration: '0' } + }) + try { + await assert.rejects( + resumeTerraformFence(generation.config, generation.deps), + /saved-plan generation/ + ) + } finally { + generation.cleanup() + } + const digest = replayHarness({ downloadedPlan: 'tampered plan' }) + try { + await assert.rejects( + resumeTerraformFence(digest.config, digest.deps), + /saved fence plan digest mismatch/ + ) + assert.equal(digest.applies(), 0) + assert.equal(digest.deleted(), false) + } finally { + digest.cleanup() + } +}) + +test('rejects Terraform state lineage or serial drift before replay', async () => { + const harness = replayHarness({ + initialProgress: { + stateTargetSize: 1, + liveTargetSize: 1, + instanceCount: 1, + operationStatus: 'ABSENT', + operationError: false, + stateLineage, + stateSerial: 8 + } + }) + try { + await assert.rejects( + resumeTerraformFence(harness.config, harness.deps), + /lineage or serial changed/ + ) + assert.equal(harness.downloads(), 0) + } finally { + harness.cleanup() + } +}) + +test('keeps the durable plan when replay cannot acquire the Terraform lock', async () => { + const notStarted = { + stateTargetSize: 1, + liveTargetSize: 1, + instanceCount: 1, + operationStatus: 'ABSENT', + operationError: false, + stateLineage, + stateSerial: 7 + } + const harness = replayHarness({ + initialProgress: notStarted, + finalProgress: notStarted, + applyError: new Error('state lock unavailable') + }) + try { + await assert.rejects( + resumeTerraformFence(harness.config, harness.deps), + /recover-forward required/ + ) + assert.equal(harness.deleted(), false) + assert.equal(harness.attested(), false) + } finally { + harness.cleanup() + } +}) + +test('deletes the exact object generation permanently and accepts confirmed absence', async () => { + const config = applyConfig() + const attempt = durableAttempt(config) + let args + await deleteTerraformFencePlan( + config, + { + commandResult: (value) => { + args = value + return { status: 0, stderr: '' } + } + }, + attempt + ) + assert.equal( + args[2], + `gs://project-terraform-state/${attempt.planObjectName}#${attempt.planObjectGeneration}` + ) + await assert.doesNotReject( + deleteTerraformFencePlan( + config, + { commandResult: () => ({ status: 1, stderr: '404 not found' }) }, + attempt + ) + ) + await assert.rejects( + deleteTerraformFencePlan( + config, + { commandResult: () => ({ status: 1, stderr: 'permission denied' }) }, + attempt + ), + /could not be deleted/ + ) +}) + +test('binds a DONE resize operation to the exact post-start audit request reason', async () => { + const config = applyConfig() + const attempt = durableAttempt(config, { applyStartedAt: 101 }) + const targetLink = + `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}` + const terraform = (args) => + args.includes('state') + ? JSON.stringify({ lineage: stateLineage, serial: 8 }) + : JSON.stringify(state(0)) + const gcloudJson = (args) => { + if (args.includes('list-instances')) return [] + if (args.includes('managed')) return { targetSize: 0 } + if (args[0] === 'compute') { + return [ + { + name: 'operation-1', + insertTime: new Date(102).toISOString(), + targetLink, + operationType: 'compute.instanceGroupManagers.resize', + status: 'DONE' + } + ] + } + return [ + { + protoPayload: { + requestMetadata: { + requestAttributes: { + reason: attempt.applyInvocations[0].requestReason + } + }, + resourceName: + `projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}`, + methodName: 'v1.compute.instanceGroupManagers.resize', + request: { size: 0 }, + response: { name: 'operation-1' } + } + } + ] + } + const progress = await inspectTerraformFenceProgress( + config, + { terraform, gcloudJson }, + attempt + ) + assert.equal(progress.operationAuditBound, true) + assert.equal(classifyTerraformFenceProgress(progress), 'complete') + const unbound = await inspectTerraformFenceProgress( + config, + { + terraform, + gcloudJson: (args) => (args[0] === 'logging' ? [] : gcloudJson(args)) + }, + attempt + ) + assert.throws( + () => classifyTerraformFenceProgress(unbound), + /ambiguous or unsafe/ + ) +}) + +test('inspects an older completed fence through exact principal and operation evidence', async () => { + const config = applyConfig() + const attempt = durableAttempt(config, { applyStartedAt: 101 }) + const gceOperation = 'operation-1' + const principalEmail = 'fence-broker@example.gserviceaccount.com' + const targetLink = + `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}` + const resourceName = + `projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}` + const terraform = (args) => + args.includes('state') + ? JSON.stringify({ lineage: stateLineage, serial: 8 }) + : JSON.stringify(state(0)) + const progress = await inspectCompletedTerraformFenceProgress( + config, + { + terraform, + gcloudJson: (args) => { + if (args.includes('list-instances')) return [] + if (args.includes('managed')) { + return { targetSize: 0, status: { isStable: true } } + } + if (args[0] === 'compute') { + return [ + { + name: gceOperation, + insertTime: new Date(102).toISOString(), + targetLink, + operationType: 'compute.instanceGroupManagers.resize', + status: 'DONE' + } + ] + } + return [ + { + timestamp: new Date(103).toISOString(), + protoPayload: { + authenticationInfo: { principalEmail }, + resourceName, + methodName: 'v1.compute.instanceGroupManagers.resize', + request: { size: '0' }, + response: { name: gceOperation } + } + } + ] + } + }, + attempt, + { gceOperation, principalEmail } + ) + assert.deepEqual(progress, { + stateTargetSize: 0, + stateLineage, + stateSerial: 8, + liveTargetSize: 0, + instanceCount: 0, + liveStable: true, + operationStatus: 'DONE', + operationError: false, + gceOperation + }) +}) + +test('adopts only the pinned completed older attempt without replaying Terraform', async () => { + const config = applyConfig() + const root = mkdtempSync(join(tmpdir(), 'relay-fence-completed-recovery-test-')) + const attempt = durableAttempt(config, { + fenceCommit: 'b'.repeat(40), + applyStartedAt: 101 + }) + const recovery = { + attemptId: attempt.attemptId, + fenceCommit: attempt.fenceCommit, + gceOperation: 'operation-1', + terraformStateSerial: 7, + planObjectGeneration: attempt.planObjectGeneration, + terraformStateObjectGeneration: '222222222', + terraformStateObjectSha256: 'c'.repeat(64), + principalEmail: 'fence-broker@example.gserviceaccount.com' + } + const events = [] + let applies = 0 + try { + await recoverSupersededCompletedTerraformFence( + config, + { + git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), + tmpdir: () => root, + assertCommittedFenceSet: async () => {}, + loadAttempt: async () => attempt, + resolvePlan: async () => ({ generation: attempt.planObjectGeneration }), + stateObjectBinding: async () => ({ + generation: recovery.terraformStateObjectGeneration, + sha256: recovery.terraformStateObjectSha256, + lineage: stateLineage, + serial: 8 + }), + downloadPlan: async (_value, path) => + writeFileSync(path, 'private saved plan', { mode: 0o600 }), + terraform: (args) => { + if (args.includes('apply')) applies++ + if (args.includes('show')) return JSON.stringify(plan()) + }, + inspectCompletedProgress: async () => ({ + stateTargetSize: 0, + stateLineage, + stateSerial: 8, + liveTargetSize: 0, + instanceCount: 0, + liveStable: true, + operationStatus: 'DONE', + operationError: false, + gceOperation: recovery.gceOperation + }), + markOperation: async (value, invocation) => { + events.push('operation') + return { attempt: value, invocation } + }, + assertZeroDiff: async () => events.push('zero-diff'), + postApplyGuard: async () => events.push('post-apply'), + attest: async () => events.push('attest'), + deletePlan: async () => events.push('delete'), + emit: () => events.push('emit') + }, + recovery + ) + assert.equal(applies, 0) + assert.deepEqual(events, [ + 'operation', + 'zero-diff', + 'post-apply', + 'attest', + 'delete', + 'emit' + ]) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('continues after an adopted fence was attested and its plan was deleted', async () => { + const config = applyConfig() + const root = mkdtempSync(join(tmpdir(), 'relay-fence-completed-retry-test-')) + const gceOperation = 'operation-1' + const attempt = durableAttempt(config, { + fenceCommit: 'b'.repeat(40), + applyStartedAt: 101, + completedAt: 200, + gceOperation + }) + const recovery = { + attemptId: attempt.attemptId, + fenceCommit: attempt.fenceCommit, + gceOperation, + terraformStateSerial: 7, + planObjectGeneration: attempt.planObjectGeneration, + terraformStateObjectGeneration: '222222222', + terraformStateObjectSha256: 'c'.repeat(64), + principalEmail: 'fence-broker@example.gserviceaccount.com' + } + let attested = false + try { + await recoverSupersededCompletedTerraformFence( + config, + { + git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), + tmpdir: () => root, + assertCommittedFenceSet: async () => {}, + loadAttempt: async () => attempt, + resolvePlan: async () => ({ generation: null }), + stateObjectBinding: async () => ({ + generation: recovery.terraformStateObjectGeneration, + sha256: recovery.terraformStateObjectSha256, + lineage: stateLineage, + serial: 8 + }), + inspectCompletedProgress: async () => ({ + stateTargetSize: 0, + stateLineage, + stateSerial: 8, + liveTargetSize: 0, + instanceCount: 0, + liveStable: true, + operationStatus: 'DONE', + operationError: false, + gceOperation + }), + markOperation: async () => { + throw new Error('must not rebind') + }, + assertZeroDiff: async () => {}, + postApplyGuard: async () => {}, + attest: async () => { + attested = true + }, + downloadPlan: async () => { + throw new Error('must not download a deleted plan') + }, + deletePlan: async () => { + throw new Error('must not delete an absent plan') + } + }, + recovery + ) + assert.equal(attested, true) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/cloud/dev/scripts/relay-load-connection-failure.mjs b/cloud/dev/scripts/relay-load-connection-failure.mjs new file mode 100644 index 00000000000..0238740b5a2 --- /dev/null +++ b/cloud/dev/scripts/relay-load-connection-failure.mjs @@ -0,0 +1,37 @@ +export function relayLoadFailureReason(error) { + const message = error instanceof Error ? error.message : String(error) + const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message) + if (tokenExchange) return `token_http_${tokenExchange[1]}` + const assignment = + /^relay assignment failed: ([1-5][0-9]{2})(?: (relay_capacity_exhausted|relay_connection_headroom_exhausted))?$/.exec( + message + ) + if (assignment?.[1] === '503' && assignment[2]) return 'assignment_capacity_exhausted' + if (assignment) return `assignment_http_${assignment[1]}` + const closed = /^control closed: ([0-9]{4})\b/.exec(message) + if (closed) return `control_close_${closed[1]}` + if (message === 'control open timeout') return 'control_open_timeout' + if (message === 'control response timeout') return 'control_response_timeout' + if (message === 'relay token exchange timeout') return 'token_timeout' + if (message === 'relay assignment timeout') return 'assignment_timeout' + if (message === 'WebSocket was closed before the connection was established') { + return 'socket_closed_before_open' + } + if (message === 'relay token exchange omitted token') return 'token_response_invalid' + if (message === 'relay assignment response invalid') return 'assignment_response_invalid' + if (message === 'expected host challenge') return 'host_challenge_invalid' + if (message === 'host proof challenge did not decrypt') return 'host_challenge_decrypt_failed' + if (message === 'expected host hello acknowledgement') return 'host_ack_invalid' + const socketResponse = /^Unexpected server response: ([1-5][0-9]{2})\b/.exec(message) + if (socketResponse) return `socket_http_${socketResponse[1]}` + if (/\b(?:ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ETIMEDOUT)\b/.test(message)) { + return 'socket_transport' + } + return 'unknown' +} + +export function discardFailedLoadSocket(socket) { + if (!socket) return + socket.on('error', () => undefined) + socket.terminate() +} diff --git a/cloud/dev/scripts/relay-load-connection-failure.test.mjs b/cloud/dev/scripts/relay-load-connection-failure.test.mjs new file mode 100644 index 00000000000..f583f7b6659 --- /dev/null +++ b/cloud/dev/scripts/relay-load-connection-failure.test.mjs @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' +import test from 'node:test' +import { + discardFailedLoadSocket, + relayLoadFailureReason +} from './relay-load-connection-failure.mjs' + +test('classifies only bounded aggregate connection failure reasons', () => { + assert.equal(relayLoadFailureReason(new Error('relay token exchange failed: 503')), 'token_http_503') + assert.equal(relayLoadFailureReason(new Error('relay assignment failed: 503')), 'assignment_http_503') + assert.equal( + relayLoadFailureReason( + new Error('relay assignment failed: 503 relay_connection_headroom_exhausted') + ), + 'assignment_capacity_exhausted' + ) + for (const status of [400, 429, 500]) { + assert.equal( + relayLoadFailureReason( + new Error(`${`relay assignment failed: ${status}`} relay_capacity_exhausted`) + ), + `assignment_http_${status}` + ) + } + assert.equal(relayLoadFailureReason(new Error('control closed: 4404 wrong cell')), 'control_close_4404') + assert.equal(relayLoadFailureReason(new Error('control open timeout')), 'control_open_timeout') + assert.equal(relayLoadFailureReason(new Error('relay token exchange timeout')), 'token_timeout') + assert.equal(relayLoadFailureReason(new Error('relay assignment timeout')), 'assignment_timeout') + assert.equal( + relayLoadFailureReason(new Error('relay token exchange omitted token')), + 'token_response_invalid' + ) + assert.equal( + relayLoadFailureReason(new Error('relay assignment response invalid')), + 'assignment_response_invalid' + ) + assert.equal( + relayLoadFailureReason(new Error('Unexpected server response: 503 Service Unavailable')), + 'socket_http_503' + ) + assert.equal(relayLoadFailureReason(new Error('connect ECONNRESET 127.0.0.1')), 'socket_transport') + assert.equal(relayLoadFailureReason(new Error('expected host challenge')), 'host_challenge_invalid') + assert.equal( + relayLoadFailureReason(new Error('host proof challenge did not decrypt')), + 'host_challenge_decrypt_failed' + ) + assert.equal(relayLoadFailureReason(new Error('expected host hello acknowledgement')), 'host_ack_invalid') + assert.equal(relayLoadFailureReason(new Error('host-sensitive detail')), 'unknown') +}) + +test('absorbs the setup error emitted while discarding a failed socket', () => { + const socket = new EventEmitter() + socket.terminate = () => socket.emit('error', new Error('closed before open')) + + assert.doesNotThrow(() => discardFailedLoadSocket(socket)) +}) diff --git a/cloud/dev/scripts/relay-load-control-peer.mjs b/cloud/dev/scripts/relay-load-control-peer.mjs new file mode 100644 index 00000000000..bb954a5d150 --- /dev/null +++ b/cloud/dev/scripts/relay-load-control-peer.mjs @@ -0,0 +1,891 @@ +import { createHash, createHmac } from 'node:crypto' +import { createRequire } from 'node:module' +import { controlPhase } from './relay-load-model.mjs' +import { discardFailedLoadSocket } from './relay-load-connection-failure.mjs' + +const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url)) +const nacl = requireFromRelay('tweetnacl') +const WebSocket = requireFromRelay('ws') +const { SignJWT } = await import(requireFromRelay.resolve('jose')) +const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import( + requireFromRelay.resolve('@orca-cloud/relay-contract') +) + +const CAPACITY_ASSIGNMENT_ERRORS = [ + 'relay_capacity_exhausted', + 'relay_connection_headroom_exhausted' +] + +function waitForOpen(socket, timeoutMs = 10_000) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => finish(new Error('control open timeout')), timeoutMs) + const finish = (error) => { + clearTimeout(timer) + socket.off('open', onOpen) + socket.off('close', onClose) + socket.off('error', onError) + if (error) reject(error) + else resolve() + } + const onOpen = () => finish() + const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`)) + const onError = (error) => finish(error) + socket.once('open', onOpen) + socket.once('close', onClose) + socket.once('error', onError) + }) +} + +function nextJson(socket, timeoutMs = 10_000) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => finish(new Error('control response timeout')), timeoutMs) + const finish = (error, value) => { + clearTimeout(timer) + socket.off('message', onMessage) + socket.off('close', onClose) + socket.off('error', onError) + if (error) reject(error) + else resolve(value) + } + const onMessage = (data) => { + try { + finish(undefined, JSON.parse(data.toString())) + } catch (error) { + finish(error) + } + } + const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`)) + const onError = (error) => finish(error) + socket.once('message', onMessage) + socket.once('close', onClose) + socket.once('error', onError) + }) +} + +function nextFrame(socket, timeoutMs = 10_000) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => finish(new Error('relay frame timeout')), timeoutMs) + const finish = (error, value) => { + clearTimeout(timer) + socket.off('message', onMessage) + socket.off('close', onClose) + socket.off('error', onError) + if (error) reject(error) + else resolve(value) + } + const onMessage = (data, binary) => finish(undefined, { bytes: Buffer.from(data), binary }) + const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`)) + const onError = (error) => finish(error) + socket.once('message', onMessage) + socket.once('close', onClose) + socket.once('error', onError) + }) +} + +function receiveBinaryStream(socket, expectedBytes, timeoutMs) { + return new Promise((resolve, reject) => { + let receivedBytes = 0 + const hash = createHash('sha256') + const timer = setTimeout(() => finish(new Error('relay stream timeout')), timeoutMs) + const finish = (error, value) => { + clearTimeout(timer) + socket.off('message', onMessage) + socket.off('close', onClose) + socket.off('error', onError) + if (error) reject(error) + else resolve(value) + } + const onMessage = (data, binary) => { + if (!binary) return finish(new Error('relay changed stream opcode')) + const bytes = Buffer.from(data) + receivedBytes += bytes.byteLength + hash.update(bytes) + if (receivedBytes > expectedBytes) return finish(new Error('relay expanded reader stream')) + if (receivedBytes === expectedBytes) { + finish(undefined, { bytes: receivedBytes, digest: hash.digest('hex') }) + } + } + const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`)) + const onError = (error) => finish(error) + socket.on('message', onMessage) + socket.once('close', onClose) + socket.once('error', onError) + }) +} + +function closeInfo(socket, timeoutMs) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => finish(new Error('reader close timeout')), timeoutMs) + const finish = (error, value) => { + clearTimeout(timer) + socket.off('close', onClose) + socket.off('error', onError) + if (error) reject(error) + else resolve(value) + } + const onClose = (code, reason) => finish(undefined, { code, reason: reason.toString() }) + const onError = (error) => finish(error) + socket.once('close', onClose) + socket.once('error', onError) + }) +} + +export function relayLoadWedgedCloseAccepted(closeCodes) { + return closeCodes.length === 2 && closeCodes[1] === 4429 && + (closeCodes[0] === 4429 || closeCodes[0] === 1006) +} + +function waitForClose(socket, timeoutMs = 10_000) { + if (!socket || socket.readyState === socket.CLOSED) return Promise.resolve() + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + socket.off('close', onClose) + discardFailedLoadSocket(socket) + reject(new Error('control close timeout')) + }, timeoutMs) + const onClose = () => { + clearTimeout(timer) + resolve() + } + socket.once('close', onClose) + }) +} + +async function cancelResponse(response) { + try { + await response.body?.cancel() + } catch { + // Preserve the bounded failure classification. + } +} + +function proofForChallenge(challenge, hostSecretKey) { + const plaintext = nacl.box.open( + Buffer.from(challenge.ciphertextB64, 'base64'), + Buffer.from(challenge.nonceB64, 'base64'), + Buffer.from(challenge.relayEphemeralPublicKeyB64, 'base64'), + hostSecretKey + ) + if (!plaintext) throw new Error('host proof challenge did not decrypt') + const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`) + const transcriptLength = new DataView( + plaintext.buffer, + plaintext.byteOffset + domain.length, + 4 + ).getUint32(0, false) + const transcriptStart = domain.length + 4 + const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength) + const secret = plaintext.slice(transcriptStart + transcriptLength) + return createHmac('sha256', secret).update(buildHostProofMacInput(transcript)).digest('base64') +} + +export class RelayLoadControlPeer { + constructor(index, options, observe) { + if (options.directorOrigin && options.targetOrigin) { + throw new Error('provide either directorOrigin or targetOrigin, not both') + } + this.index = index + this.options = options + this.observe = observe + this.keys = nacl.box.keyPair() + this.relayHostId = createHash('sha256') + .update(this.keys.publicKey) + .digest('base64url') + .slice(0, 16) + this.phase = controlPhase(index, options.seed) + this.socket = null + this.generation = undefined + this.controlResumeSecret = undefined + this.lastAssignment = undefined + this.refreshTimer = null + this.stopped = false + this.connecting = false + this.inFlight = new Set() + this.shutdownPromise = null + this.abortController = new AbortController() + this.drainExpected = false + this.controlWaiters = new Set() + this.spliceSockets = new Set() + this.spliceSequence = 0 + } + + connect() { + if ( + this.stopped || + this.connecting || + (this.socket !== null && this.socket.readyState === this.socket.OPEN) + ) { + return Promise.resolve() + } + this.connecting = true + const operation = this.connectOnce() + this.inFlight.add(operation) + const finish = () => { + this.connecting = false + this.inFlight.delete(operation) + } + operation.then(finish, finish) + return operation + } + + assignedCellUrl() { + return this.lastAssignment?.cellUrl + } + + async connectOnce() { + let socket = null + try { + const relayToken = await this.relayToken() + if (this.stopped) return + const assignment = await this.assignment(relayToken) + if (this.stopped) return + this.lastAssignment = assignment + socket = this.createSocket(assignment, relayToken) + this.socket = socket + this.drainExpected = false + await waitForOpen(socket) + if (this.stopped || this.socket !== socket) return + socket.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: this.relayHostId, + assignmentEpoch: assignment.assignmentEpoch, + hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'), + appVersion: 'relay-load', + ...(this.generation === undefined ? {} : { previousGeneration: this.generation }), + ...(this.controlResumeSecret === undefined + ? {} + : { controlResumeSecret: this.controlResumeSecret }) + }) + ) + const challenge = await nextJson(socket) + if (this.stopped || this.socket !== socket) return + if (challenge.type !== 'host-challenge') throw new Error('expected host challenge') + socket.send( + JSON.stringify({ + type: 'host-challenge-ack', + challengeId: challenge.challengeId, + proofB64: proofForChallenge(challenge, this.keys.secretKey) + }) + ) + const ack = await nextJson(socket) + if (this.stopped || this.socket !== socket) return + if (ack.type !== 'host-hello-ack') throw new Error('expected host hello acknowledgement') + this.generation = ack.generation + this.controlResumeSecret = ack.controlResumeSecret + socket.on('message', (data) => this.onMessage(socket, data)) + socket.once('close', (code) => this.onClose(socket, code)) + socket.once('error', (error) => this.observe('socketError', { index: this.index, error })) + this.observe('connected', { index: this.index }) + this.scheduleRefresh(this.phase.refreshOffsetMs) + } catch (error) { + discardFailedLoadSocket(socket) + if (this.socket === socket) this.socket = null + if (this.stopped) return + throw error + } + } + + createSocket(assignment, relayToken) { + return new WebSocket(`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`, { + headers: { authorization: `Bearer ${relayToken}` }, + perMessageDeflate: false + }) + } + + shutdown() { + if (this.shutdownPromise) return this.shutdownPromise + this.stopped = true + this.abortController.abort() + if (this.refreshTimer) clearTimeout(this.refreshTimer) + this.refreshTimer = null + this.shutdownPromise = this.shutdownOnce() + return this.shutdownPromise + } + + async shutdownOnce() { + const socket = this.socket + const closed = waitForClose(socket) + for (const spliceSocket of this.spliceSockets) { + if ( + spliceSocket.readyState !== spliceSocket.CLOSED && + spliceSocket.readyState !== spliceSocket.CLOSING + ) { + spliceSocket.close(1000, 'load complete') + } + } + this.rejectControlWaiters(new Error('control stopped')) + if (socket && socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) { + socket.close(1000, 'load complete') + } + const settled = async () => { + while (this.inFlight.size > 0) { + await Promise.allSettled([...this.inFlight]) + } + } + await Promise.all([closed, settled()]) + this.observe('shutdown', { + index: this.index, + activeControls: this.socket?.readyState === this.socket?.OPEN ? 1 : 0, + activeSpliceSockets: this.spliceSockets.size, + inFlightOperations: this.inFlight.size, + refreshTimerActive: this.refreshTimer !== null + }) + } + + openSplice(options = {}) { + if (this.stopped) return Promise.reject(new Error('control stopped')) + const operation = this.openSpliceOnce(options) + this.inFlight.add(operation) + const finish = () => this.inFlight.delete(operation) + operation.then(finish, finish) + return operation + } + + openInviteOffer() { + if (this.stopped) return Promise.reject(new Error('control stopped')) + const operation = this.openInviteOfferOnce() + this.inFlight.add(operation) + const finish = () => this.inFlight.delete(operation) + operation.then(finish, finish) + return operation + } + + async openInviteOfferOnce() { + if (!this.socket || this.socket.readyState !== this.socket.OPEN) { + throw new Error('active control required for invite offer') + } + const sequence = this.spliceSequence++ + const reqId = `load-offer-${this.index}-${sequence}` + const response = this.waitForControlMessage( + (message) => + message.reqId === reqId && + (message.type === 'invite-created' || message.type === 'control-error') + ) + this.socket.send(JSON.stringify({ + type: 'invite-create', + reqId, + relayDeviceId: `load-offer-device-${this.index}-${sequence}` + })) + const result = await response + if (result.type === 'control-error') throw new Error(`invite offer failed: ${result.code}`) + if ( + typeof result.inviteToken !== 'string' || + !Number.isSafeInteger(result.expiresAt) || + result.expiresAt <= Date.now() + ) throw new Error('relay invite offer response invalid') + } + + async openSpliceOnce({ + payloadBytes = 64, + readerMode = 'normal', + readerHoldMs = 0, + streamBytes = payloadBytes, + frameBytes = payloadBytes, + observeReaderPressure = async () => undefined, + readerDelay = async (ms) => await new Promise((resolve) => setTimeout(resolve, ms)), + slowReaderHoldMs = 0, + holdMs = 0 + } = {}) { + if ( + !this.socket || + this.socket.readyState !== this.socket.OPEN || + this.generation === undefined || + this.lastAssignment === undefined + ) { + throw new Error('active control required for splice') + } + if (!Number.isSafeInteger(payloadBytes) || payloadBytes < 1) { + throw new Error('splice payload bytes must be positive') + } + const sequence = this.spliceSequence++ + const reqId = `load-invite-${this.index}-${sequence}` + const relayDeviceId = `load-device-${this.index}-${sequence}` + let phone + let data + let opened = false + try { + const invitePromise = this.waitForControlMessage( + (message) => message.type === 'invite-created' && message.reqId === reqId + ) + this.socket.send(JSON.stringify({ type: 'invite-create', reqId, relayDeviceId })) + const invite = await invitePromise + if (typeof invite.inviteToken !== 'string') throw new Error('relay invite response invalid') + + phone = this.createClientSocket(this.lastAssignment) + this.trackSpliceSocket(phone) + await waitForOpen(phone) + const connectionPromise = this.waitForControlMessage( + (message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId + ) + phone.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) + ) + const connection = await connectionPromise + if (typeof connection.connId !== 'string' || typeof connection.connTicket !== 'string') { + throw new Error('relay connection response invalid') + } + + data = this.createHostDataSocket(this.lastAssignment, connection.connId) + this.trackSpliceSocket(data) + await waitForOpen(data) + const phoneHello = nextJson(phone) + data.send( + JSON.stringify({ + type: 'host-data-auth', + v: 1, + connTicket: connection.connTicket, + generation: this.generation + }) + ) + if ((await phoneHello).ok !== true) throw new Error('relay rejected load splice') + + if (slowReaderHoldMs > 0 && readerMode === 'normal') { + readerMode = 'slow' + readerHoldMs = slowReaderHoldMs + streamBytes = payloadBytes + frameBytes = payloadBytes + } + if (!['normal', 'slow', 'wedged'].includes(readerMode)) { + throw new Error('reader mode is invalid') + } + const pausedSocket = readerMode === 'normal' ? undefined : phone._socket + if (readerMode !== 'normal' && !pausedSocket) throw new Error('reader transport unavailable') + if (readerMode === 'wedged') { + const closes = [closeInfo(phone, readerHoldMs + 10_000), closeInfo(data, readerHoldMs + 10_000)] + pausedSocket.pause() + const readerPausedAt = Date.now() + const readerPressure = observeReaderPressure({ + cellOrigin: this.lastAssignment.cellUrl, + readerMode, + streamBytes + }) + const [sent] = await Promise.all([ + this.sendReaderStream(data, sequence, streamBytes, frameBytes, readerDelay), + readerPressure + ]) + await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt))) + pausedSocket.resume() + const closeEvidence = await Promise.all(closes) + const closeCodes = closeEvidence.map(({ code }) => code) + if (!relayLoadWedgedCloseAccepted(closeCodes)) { + throw new Error(`wedged reader close codes: ${closeCodes.join(',')}`) + } + opened = true + this.observe('spliceOpened', { index: this.index, readerMode }) + this.observe('spliceWedged', { index: this.index, code: 4429, streamBytes: sent.bytes }) + return + } + + const expectedStreamBytes = readerMode === 'slow' ? streamBytes : payloadBytes + const expectedFrameBytes = readerMode === 'slow' ? frameBytes : payloadBytes + const phoneStream = receiveBinaryStream( + phone, + expectedStreamBytes, + readerMode === 'slow' ? readerHoldMs + 10_000 : 10_000 + ) + const readerPausedAt = pausedSocket ? Date.now() : 0 + if (pausedSocket) pausedSocket.pause() + const readerPressure = readerMode === 'slow' + ? observeReaderPressure({ + cellOrigin: this.lastAssignment.cellUrl, + readerMode, + streamBytes: expectedStreamBytes + }) + : Promise.resolve() + const [sent] = await Promise.all([ + this.sendReaderStream( + data, + sequence, + expectedStreamBytes, + expectedFrameBytes, + readerDelay + ), + readerPressure + ]) + if (readerMode === 'slow') { + await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt))) + pausedSocket.resume() + } + const receivedByPhone = await phoneStream + if (receivedByPhone.bytes !== sent.bytes || receivedByPhone.digest !== sent.digest) { + throw new Error('relay changed host-to-client splice payload') + } + + const textPayload = `orca-relay-load:${this.index}:${sequence}:${payloadBytes}` + const dataFrame = nextFrame(data) + phone.send(textPayload) + const receivedByHost = await dataFrame + if (receivedByHost.binary || receivedByHost.bytes.toString() !== textPayload) { + throw new Error('relay changed client-to-host splice payload') + } + opened = true + this.observe('spliceOpened', { index: this.index, readerMode }) + if (holdMs > 0 && !(await this.waitForSpliceHold(holdMs, [phone, data]))) return + this.observe('spliceCompleted', { index: this.index, readerMode }) + } catch (error) { + if (!this.stopped) this.observe('spliceFailed', { index: this.index, error }) + throw error + } finally { + await Promise.all([this.closeSpliceSocket(phone), this.closeSpliceSocket(data)]) + if (opened) this.observe('spliceClosed', { index: this.index }) + } + } + + waitForSpliceHold(holdMs, sockets) { + if (this.stopped) return Promise.resolve(false) + return new Promise((resolve, reject) => { + const finish = (error, completed = false) => { + clearTimeout(timer) + this.abortController.signal.removeEventListener('abort', onAbort) + for (const socket of sockets) { + socket.off('close', onClose) + socket.off('error', onError) + } + if (error) reject(error) + else resolve(completed) + } + const onAbort = () => finish(undefined, false) + const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`)) + const onError = (error) => finish(error) + const timer = setTimeout(() => finish(undefined, true), holdMs) + this.abortController.signal.addEventListener('abort', onAbort, { once: true }) + for (const socket of sockets) { + socket.once('close', onClose) + socket.once('error', onError) + } + }) + } + + createClientSocket(assignment) { + return new WebSocket( + `${assignment.cellUrl.replace(/^http/, 'ws')}/v1/connect/${this.relayHostId}`, + { perMessageDeflate: false } + ) + } + + createHostDataSocket(assignment, connId) { + return new WebSocket( + `${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/data/${connId}`, + { perMessageDeflate: false } + ) + } + + splicePayload(sequence, payloadBytes) { + const seed = createHash('sha256') + .update(`orca-relay-load:${this.index}:${sequence}`) + .digest() + return Buffer.allocUnsafe(payloadBytes).map((_, index) => seed[index % seed.length]) + } + + async sendReaderStream(socket, sequence, streamBytes, frameBytes, delay) { + const hash = createHash('sha256') + let sentBytes = 0 + let frameIndex = 0 + while (sentBytes < streamBytes) { + const bytes = Math.min(frameBytes, streamBytes - sentBytes) + const payload = this.splicePayload(sequence + frameIndex, bytes) + await this.sendReaderFrame(socket, payload) + hash.update(payload) + sentBytes += bytes + frameIndex++ + while (socket.bufferedAmount > frameBytes) await delay(10) + } + return { bytes: sentBytes, digest: hash.digest('hex') } + } + + sendReaderFrame(socket, payload) { + if (socket.send.length < 2) { + socket.send(payload) + return Promise.resolve() + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('reader send timeout')), 10_000) + socket.send(payload, (error) => { + clearTimeout(timer) + if (error) reject(error) + else resolve() + }) + }) + } + + trackSpliceSocket(socket) { + this.spliceSockets.add(socket) + socket.once('close', () => this.spliceSockets.delete(socket)) + } + + async closeSpliceSocket(socket) { + if (!socket) return + const closed = waitForClose(socket).catch(() => undefined) + if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) { + socket.close(1000, 'splice complete') + } + await closed + this.spliceSockets.delete(socket) + } + + async openRebindProbe() { + if ( + !this.socket || + this.socket.readyState !== this.socket.OPEN || + this.generation === undefined || + this.controlResumeSecret === undefined || + this.lastAssignment === undefined + ) { + throw new Error('active control required for rebind probe') + } + const relayToken = await this.relayToken() + const socket = new WebSocket( + `${this.lastAssignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`, + { + headers: { authorization: `Bearer ${relayToken}` }, + perMessageDeflate: false + } + ) + try { + await waitForOpen(socket) + socket.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId: this.relayHostId, + assignmentEpoch: this.lastAssignment.assignmentEpoch, + hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'), + appVersion: 'relay-load-rebind-proof', + previousGeneration: this.generation, + controlResumeSecret: this.controlResumeSecret + }) + ) + const challenge = await nextJson(socket) + if (challenge.type !== 'host-challenge') throw new Error('expected host challenge') + socket.on('error', () => undefined) + const closed = new Promise((resolve) => socket.once('close', resolve)) + return { + close: async () => { + const closeCompleted = waitForClose(socket) + if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) { + socket.close(1000, 'rebind boundary proved') + } + await closeCompleted + }, + closed, + isOpen: () => socket.readyState === socket.OPEN + } + } catch (error) { + discardFailedLoadSocket(socket) + await waitForClose(socket).catch(() => undefined) + throw error + } + } + + async relayToken() { + const accessToken = this.options.accessTokenProvider + ? await this.options.accessTokenProvider() + : this.options.accessToken + if (accessToken) { + const body = await this.requestJson( + `${this.options.authOrigin}/v1/desktop/auth/relay-token`, + { + method: 'POST', + headers: { + authorization: `Bearer ${accessToken}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + relayHostId: this.relayHostId, + hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64') + }) + }, + 'relay token exchange timeout', + (status) => `relay token exchange failed: ${status}` + ) + if (typeof body.relayToken !== 'string') throw new Error('relay token exchange omitted token') + if (!this.stopped) this.observe('token', { index: this.index }) + return body.relayToken + } + const token = await new SignJWT({ + prof: `load-profile-${this.index}`, + org: 'relay-load', + purpose: 'host-control', + relayHostId: this.relayHostId + }) + .setProtectedHeader({ alg: 'ES256', kid: this.options.signingKeyId }) + .setIssuer(this.options.authOrigin) + .setAudience('orca-relay') + .setSubject(`load-user-${this.index}`) + .setIssuedAt() + .setExpirationTime('5m') + .sign(this.options.signingKey) + if (!this.stopped) this.observe('token', { index: this.index }) + return token + } + + async requestAssignment(preferredRegion) { + return await this.assignment(await this.relayToken(), preferredRegion) + } + + async assignment(relayToken, preferredRegion = this.options.preferredRegion) { + if (!this.options.directorOrigin) { + if (!this.options.targetOrigin) throw new Error('relay target origin missing') + return { cellUrl: this.options.targetOrigin, assignmentEpoch: 1 } + } + const body = await this.requestJson( + `${this.options.directorOrigin}/v1/assign`, + { + method: 'POST', + headers: { authorization: `Bearer ${relayToken}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + v: 1, + relayHostId: this.relayHostId, + ...(preferredRegion ? { preferredRegion } : {}) + }) + }, + 'relay assignment timeout', + (status, errorCode) => + `relay assignment failed: ${status}${errorCode ? ` ${errorCode}` : ''}`, + CAPACITY_ASSIGNMENT_ERRORS + ) + if ( + typeof body.cellUrl !== 'string' || + !Number.isSafeInteger(body.assignmentEpoch) || + body.assignmentEpoch < 1 + ) { + throw new Error('relay assignment response invalid') + } + return body + } + + async requestJson(url, init, timeoutMessage, httpErrorMessage, allowedErrorCodes = []) { + const controller = new AbortController() + const onShutdown = () => controller.abort() + if (this.abortController.signal.aborted) controller.abort() + else this.abortController.signal.addEventListener('abort', onShutdown, { once: true }) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + controller.abort() + }, this.options.requestTimeoutMs ?? 10_000) + try { + const response = await fetch(url, { ...init, signal: controller.signal }) + if (!response.ok) { + let bodyConsumed = false + let errorCode + if (allowedErrorCodes.length > 0) { + try { + const body = await response.json() + bodyConsumed = true + if (allowedErrorCodes.includes(body?.error)) errorCode = body.error + } catch { + // Preserve the bounded status-only classification. + } + } + if (!bodyConsumed) await cancelResponse(response) + throw new Error(httpErrorMessage(response.status, errorCode)) + } + return await response.json() + } catch (error) { + if (timedOut) throw new Error(timeoutMessage, { cause: error }) + throw error + } finally { + clearTimeout(timer) + this.abortController.signal.removeEventListener('abort', onShutdown) + } + } + + onMessage(socket, data) { + let message + try { + message = JSON.parse(data.toString()) + } catch { + this.observe('protocolError', { index: this.index }) + return + } + for (const waiter of this.controlWaiters) { + if (waiter.matches(message)) { + this.controlWaiters.delete(waiter) + clearTimeout(waiter.timer) + waiter.resolve(message) + return + } + } + if (message.type === 'ping') { + socket.send(JSON.stringify({ type: 'pong', t: message.t })) + this.observe('ping', { index: this.index }) + } else if (message.type === 'drain') { + this.drainExpected = true + this.observe('drain', { index: this.index }) + } + } + + onClose(socket, code) { + if (this.socket !== socket) return + this.socket = null + if (this.refreshTimer) clearTimeout(this.refreshTimer) + this.refreshTimer = null + this.rejectControlWaiters(new Error(`control closed: ${code}`)) + this.observe('closed', { + index: this.index, + code, + stopped: this.stopped, + expectedDrain: this.drainExpected + }) + this.drainExpected = false + } + + waitForControlMessage(matches, timeoutMs = 10_000) { + if (this.stopped) return Promise.reject(new Error('control stopped')) + return new Promise((resolve, reject) => { + const waiter = { + matches, + resolve, + reject, + timer: setTimeout(() => { + this.controlWaiters.delete(waiter) + reject(new Error('control response timeout')) + }, timeoutMs) + } + this.controlWaiters.add(waiter) + }) + } + + rejectControlWaiters(error) { + for (const waiter of this.controlWaiters) { + clearTimeout(waiter.timer) + waiter.reject(error) + } + this.controlWaiters.clear() + } + + scheduleRefresh(delayMs) { + if (this.stopped) return + this.refreshTimer = setTimeout(() => { + void this.refresh().then( + () => this.scheduleRefresh(this.phase.refreshIntervalMs), + () => this.scheduleRefresh(this.phase.refreshIntervalMs) + ) + }, delayMs) + } + + refresh() { + if (this.stopped) return Promise.resolve() + const operation = this.refreshOnce() + this.inFlight.add(operation) + const finish = () => this.inFlight.delete(operation) + operation.then(finish, finish) + return operation + } + + async refreshOnce() { + const socket = this.socket + if (this.stopped || !socket || socket.readyState !== socket.OPEN) return + try { + const relayJwt = await this.relayToken() + if (this.stopped || this.socket !== socket || socket.readyState !== socket.OPEN) return + socket.send(JSON.stringify({ type: 'auth-refresh', relayJwt })) + this.observe('refresh', { index: this.index }) + } catch (error) { + if (!this.stopped) this.observe('refreshError', { index: this.index, error }) + } + } +} diff --git a/cloud/dev/scripts/relay-load-control-peer.test.mjs b/cloud/dev/scripts/relay-load-control-peer.test.mjs new file mode 100644 index 00000000000..95ebd89e1ef --- /dev/null +++ b/cloud/dev/scripts/relay-load-control-peer.test.mjs @@ -0,0 +1,903 @@ +import assert from 'node:assert/strict' +import { generateKeyPairSync } from 'node:crypto' +import { EventEmitter } from 'node:events' +import { createRequire } from 'node:module' +import test from 'node:test' +import { + RelayLoadControlPeer, + relayLoadWedgedCloseAccepted +} from './relay-load-control-peer.mjs' + +const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url)) +const nacl = requireFromRelay('tweetnacl') +const { buildHostChallengePlaintext } = await import( + requireFromRelay.resolve('@orca-cloud/relay-contract') +) + +function deferred() { + let resolve + let reject + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +function peerOptions(overrides = {}) { + const { privateKey } = generateKeyPairSync('ec', { namedCurve: 'P-256' }) + return { + authOrigin: 'https://auth.test', + directorOrigin: 'https://director.test', + reconnectMaxMs: 0, + seed: 1, + signingKey: privateKey, + signingKeyId: 'test-key', + ...overrides + } +} + +function response(body) { + return { ok: true, status: 200, json: async () => body } +} + +function fakeOpenSocket() { + const socket = new EventEmitter() + socket.OPEN = 1 + socket.CLOSING = 2 + socket.CLOSED = 3 + socket.readyState = socket.OPEN + socket.sent = [] + socket.send = (message) => socket.sent.push(message) + socket.close = (code = 1000, reason = '') => { + socket.readyState = socket.CLOSING + queueMicrotask(() => { + socket.readyState = socket.CLOSED + socket.emit('close', code, Buffer.from(reason)) + }) + } + socket.terminate = socket.close + return socket +} + +function fakeHandshakeSocket() { + const socket = fakeOpenSocket() + socket.CONNECTING = 0 + socket.readyState = socket.CONNECTING + socket.open = () => { + socket.readyState = socket.OPEN + socket.emit('open') + } + socket.message = (message) => socket.emit('message', Buffer.from(JSON.stringify(message))) + return socket +} + +function openOnNextTurn(socket) { + queueMicrotask(() => socket.open()) + return socket +} + +function validChallenge(peer) { + const relayKeys = nacl.box.keyPair() + const nonce = nacl.randomBytes(nacl.box.nonceLength) + const plaintext = buildHostChallengePlaintext( + new Uint8Array([1, 2, 3]), + nacl.randomBytes(32) + ) + const ciphertext = nacl.box(plaintext, nonce, peer.keys.publicKey, relayKeys.secretKey) + return { + type: 'host-challenge', + challengeId: 'test-challenge', + ciphertextB64: Buffer.from(ciphertext).toString('base64'), + nonceB64: Buffer.from(nonce).toString('base64'), + relayEphemeralPublicKeyB64: Buffer.from(relayKeys.publicKey).toString('base64') + } +} + +test('shutdown waits for pending assignment and prevents a late connection', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + const assignment = deferred() + const assignmentStarted = deferred() + global.fetch = () => { + assignmentStarted.resolve() + return assignment.promise + } + const observations = [] + const peer = new RelayLoadControlPeer(0, peerOptions(), (type) => observations.push(type)) + + const connecting = peer.connect() + await assignmentStarted.promise + let shutdownFinished = false + const shutdown = peer.shutdown().then(() => { + shutdownFinished = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(shutdownFinished, false) + + assignment.resolve(response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 })) + await Promise.all([connecting, shutdown]) + assert.equal(peer.socket, null) + assert.equal(observations.includes('connected'), false) +}) + +test('shutdown after socket open prevents a late handshake', async () => { + const observations = [] + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }), + (type) => observations.push(type) + ) + const socket = fakeHandshakeSocket() + const socketCreated = deferred() + peer.createSocket = () => { + socketCreated.resolve() + return socket + } + + const connecting = peer.connect() + await socketCreated.promise + socket.open() + await Promise.all([connecting, peer.shutdown()]) + + assert.deepEqual(socket.sent, []) + assert.equal(observations.includes('connected'), false) +}) + +test('shutdown after challenge prevents a late acknowledgement', async () => { + const observations = [] + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }), + (type) => observations.push(type) + ) + const socket = fakeHandshakeSocket() + const socketCreated = deferred() + const helloSent = deferred() + socket.send = (message) => { + socket.sent.push(message) + helloSent.resolve() + } + peer.createSocket = () => { + socketCreated.resolve() + return socket + } + + const connecting = peer.connect() + await socketCreated.promise + socket.open() + await helloSent.promise + socket.message({ type: 'host-challenge' }) + await Promise.all([connecting, peer.shutdown()]) + + assert.equal(socket.sent.length, 1) + assert.equal(observations.includes('connected'), false) +}) + +test('shutdown after host acknowledgement prevents a late connected observation', async () => { + const observations = [] + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }), + (type) => observations.push(type) + ) + const socket = fakeHandshakeSocket() + const socketCreated = deferred() + const helloSent = deferred() + const proofSent = deferred() + socket.send = (message) => { + socket.sent.push(message) + if (socket.sent.length === 1) helloSent.resolve() + else proofSent.resolve() + } + peer.createSocket = () => { + socketCreated.resolve() + return socket + } + + const connecting = peer.connect() + await socketCreated.promise + socket.open() + await helloSent.promise + socket.message(validChallenge(peer)) + await proofSent.promise + socket.message({ type: 'host-hello-ack', generation: 1, controlResumeSecret: 'test-secret' }) + await Promise.all([connecting, peer.shutdown()]) + + assert.equal(socket.sent.length, 2) + assert.equal(observations.includes('connected'), false) +}) + +test('shutdown prevents a pending refresh from sending or reporting success', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + const token = deferred() + const tokenStarted = deferred() + global.fetch = () => { + tokenStarted.resolve() + return token.promise + } + const observations = [] + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), + (type) => observations.push(type) + ) + const socket = fakeOpenSocket() + peer.socket = socket + + const refreshing = peer.refresh() + await tokenStarted.promise + const shutdown = peer.shutdown() + token.resolve(response({ relayToken: 'test-relay-token' })) + await Promise.all([refreshing, shutdown]) + + assert.deepEqual(socket.sent, []) + assert.equal(observations.includes('refresh'), false) + assert.equal(observations.includes('refreshError'), false) +}) + +test('shutdown suppresses a late refresh error', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + const token = deferred() + const tokenStarted = deferred() + global.fetch = () => { + tokenStarted.resolve() + return token.promise + } + const observations = [] + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), + (type) => observations.push(type) + ) + peer.socket = fakeOpenSocket() + + const refreshing = peer.refresh() + await tokenStarted.promise + const shutdown = peer.shutdown() + token.reject(new Error('late token failure')) + await Promise.all([refreshing, shutdown]) + + assert.equal(observations.includes('refreshError'), false) +}) + +test('shutdown aborts an HTTP request that would otherwise remain pending', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + const requestStarted = deferred() + let aborted = false + global.fetch = (_url, { signal }) => + new Promise((_resolve, reject) => { + requestStarted.resolve() + signal.addEventListener( + 'abort', + () => { + aborted = true + reject(new Error('request aborted')) + }, + { once: true } + ) + }) + const observations = [] + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), + (type) => observations.push(type) + ) + + const connecting = peer.connect() + await requestStarted.promise + await Promise.all([connecting, peer.shutdown()]) + + assert.equal(aborted, true) + assert.equal(observations.includes('connected'), false) +}) + +test('bounds a pending HTTP request with a classified timeout', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + global.fetch = (_url, { signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('request aborted')), { once: true }) + }) + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ + accessToken: 'test-access-token', + directorOrigin: undefined, + requestTimeoutMs: 1 + }), + () => undefined + ) + + await assert.rejects(peer.connect(), /relay token exchange timeout/) + await peer.shutdown() +}) + +test('bounds a stalled token response body', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + global.fetch = async (_url, { signal }) => ({ + ok: true, + status: 200, + json: () => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true }) + }) + }) + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ + accessToken: 'test-access-token', + directorOrigin: undefined, + requestTimeoutMs: 1 + }), + () => undefined + ) + + await assert.rejects(peer.connect(), /relay token exchange timeout/) + await peer.shutdown() +}) + +test('bounds a stalled assignment response body', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + global.fetch = async (_url, { signal }) => ({ + ok: true, + status: 200, + json: () => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true }) + }) + }) + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ requestTimeoutMs: 1 }), + () => undefined + ) + + await assert.rejects(peer.connect(), /relay assignment timeout/) + await peer.shutdown() +}) + +test('shutdown aborts a stalled successful response body', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + const bodyStarted = deferred() + let bodyAborted = false + global.fetch = async (_url, { signal }) => ({ + ok: true, + status: 200, + json: () => + new Promise((_resolve, reject) => { + bodyStarted.resolve() + signal.addEventListener( + 'abort', + () => { + bodyAborted = true + reject(new Error('body aborted')) + }, + { once: true } + ) + }) + }) + const observations = [] + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), + (type) => observations.push(type) + ) + + const connecting = peer.connect() + await bodyStarted.promise + await Promise.all([connecting, peer.shutdown()]) + + assert.equal(bodyAborted, true) + assert.equal(observations.includes('connected'), false) +}) + +test('cancels a rejected HTTP response body', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + let canceled = false + global.fetch = async () => ({ + ok: false, + status: 503, + body: { + cancel: async () => { + canceled = true + } + } + }) + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), + () => undefined + ) + + await assert.rejects(peer.connect(), /relay token exchange failed: 503/) + await peer.shutdown() + assert.equal(canceled, true) +}) + +test('preserves only an exact capacity assignment rejection reason', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + global.fetch = async () => ({ + ok: false, + status: 503, + json: async () => ({ error: 'relay_connection_headroom_exhausted' }) + }) + const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined) + + await assert.rejects( + peer.connect(), + /relay assignment failed: 503 relay_connection_headroom_exhausted/ + ) + await peer.shutdown() +}) + +test('sends preferred region and preserves the genuine director epoch', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + let assignmentRequest + global.fetch = async (_url, init) => { + assignmentRequest = JSON.parse(init.body) + return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 47 }) + } + const peer = new RelayLoadControlPeer( + 0, + peerOptions({ preferredRegion: 'asia-east2' }), + () => undefined + ) + + const assignment = await peer.assignment('relay-token') + + assert.deepEqual(assignmentRequest, { + v: 1, + relayHostId: peer.relayHostId, + preferredRegion: 'asia-east2' + }) + assert.equal(assignment.assignmentEpoch, 47) + await peer.shutdown() +}) + +test('omits preferred region and rejects a fabricated director epoch', async (context) => { + const originalFetch = global.fetch + context.after(() => { + global.fetch = originalFetch + }) + let assignmentRequest + global.fetch = async (_url, init) => { + assignmentRequest = JSON.parse(init.body) + return response({ cellUrl: 'https://cell.test', assignmentEpoch: 0 }) + } + const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined) + + await assert.rejects(peer.assignment('relay-token'), /assignment response invalid/) + assert.equal('preferredRegion' in assignmentRequest, false) + await peer.shutdown() +}) + +test('rejects ambiguous direct and director assignment modes', () => { + assert.throws( + () => + new RelayLoadControlPeer( + 0, + peerOptions({ targetOrigin: 'https://cell.test' }), + () => undefined + ), + /either directorOrigin or targetOrigin/ + ) +}) + +test('opens a genuine splice and verifies payloads in both directions', async () => { + const observations = [] + const peer = new RelayLoadControlPeer(3, peerOptions(), (type, detail) => { + observations.push({ type, detail }) + }) + const control = fakeOpenSocket() + const phone = fakeHandshakeSocket() + const data = fakeHandshakeSocket() + let dataPayloadBytes = 0 + let observationStarted = false + let sentBeforeObservation = false + let paused = 0 + let resumed = 0 + phone._socket = { + pause: () => paused++, + resume: () => resumed++ + } + control.send = (raw) => { + const message = JSON.parse(raw) + if (message.type === 'invite-create') { + queueMicrotask(() => + control.emit( + 'message', + Buffer.from( + JSON.stringify({ + type: 'invite-created', + reqId: message.reqId, + inviteToken: 'invite-token' + }) + ) + ) + ) + } + } + phone.send = (raw) => { + if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') { + queueMicrotask(() => + control.emit( + 'message', + Buffer.from( + JSON.stringify({ + type: 'conn-open', + relayDeviceId: 'load-device-3-0', + connId: 'connection-1', + connTicket: 'connection-ticket' + }) + ) + ) + ) + return + } + queueMicrotask(() => data.emit('message', Buffer.from(raw), false)) + } + data.send = (raw) => { + if (typeof raw === 'string') { + queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) + return + } + const dataPayload = Buffer.from(raw) + if (!observationStarted) sentBeforeObservation = true + dataPayloadBytes += dataPayload.byteLength + queueMicrotask(() => phone.emit('message', dataPayload, true)) + } + peer.socket = control + peer.generation = 9 + peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 47 } + control.on('message', (raw) => peer.onMessage(control, raw)) + peer.createClientSocket = () => openOnNextTurn(phone) + peer.createHostDataSocket = () => openOnNextTurn(data) + + await peer.openSplice({ + readerMode: 'slow', + readerHoldMs: 1, + streamBytes: 300 * 1024, + frameBytes: 64 * 1024, + observeReaderPressure: async () => { observationStarted = true } + }) + + assert.equal(dataPayloadBytes, 300 * 1024) + assert.equal(sentBeforeObservation, false) + assert.equal(paused, 1) + assert.equal(resumed, 1) + assert.equal(observations.filter(({ type }) => type === 'spliceCompleted').length, 1) + assert.equal(observations.some(({ type }) => type === 'spliceFailed'), false) + await peer.shutdown() + assert.deepEqual(observations.at(-1), { + type: 'shutdown', + detail: { + index: 3, + activeControls: 0, + activeSpliceSockets: 0, + inFlightOperations: 0, + refreshTimerActive: false + } + }) +}) + +test('opens invitation leases and preserves exact capacity errors', async () => { + const peer = new RelayLoadControlPeer(4, peerOptions(), () => undefined) + const control = fakeOpenSocket() + peer.socket = control + control.on('message', (raw) => peer.onMessage(control, raw)) + let calls = 0 + control.send = (raw) => { + const request = JSON.parse(raw) + calls++ + queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify( + calls === 1 + ? { + type: 'invite-created', reqId: request.reqId, + inviteToken: 'invite-token', expiresAt: Date.now() + 60_000 + } + : { type: 'control-error', reqId: request.reqId, code: 'relay_capacity_exhausted' } + )))) + } + + await peer.openInviteOffer() + await assert.rejects(peer.openInviteOffer(), /invite offer failed: relay_capacity_exhausted/) + await peer.shutdown() +}) + +test('can request the same assignment with an explicit replacement preference', async (context) => { + const originalFetch = global.fetch + context.after(() => { global.fetch = originalFetch }) + const requests = [] + global.fetch = async (url, init) => { + if (String(url).endsWith('/v1/assign')) { + requests.push(JSON.parse(init.body)) + return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 3 }) + } + return response({ relayToken: 'relay-token' }) + } + const peer = new RelayLoadControlPeer( + 5, + peerOptions({ accessToken: 'access-token', preferredRegion: 'asia-east2' }), + () => undefined + ) + + assert.equal((await peer.requestAssignment('us-central1')).cellUrl, 'https://asia-cell.test') + assert.equal(requests[0].preferredRegion, 'us-central1') + await peer.shutdown() +}) + +test('uses a refreshable workflow access-token provider', async (context) => { + const originalFetch = global.fetch + context.after(() => { global.fetch = originalFetch }) + let providerCalls = 0 + let authorization + global.fetch = async (url, init) => { + if (String(url).endsWith('/v1/desktop/auth/relay-token')) { + authorization = init.headers.authorization + return response({ relayToken: 'relay-token' }) + } + return response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 }) + } + const peer = new RelayLoadControlPeer(6, peerOptions({ + accessTokenProvider: async () => { + providerCalls++ + return 'refreshed-access-token' + } + }), () => undefined) + + await peer.requestAssignment('asia-east2') + assert.equal(providerCalls, 1) + assert.equal(authorization, 'Bearer refreshed-access-token') + await peer.shutdown() +}) + +test('accepts a forced close only when the responsive splice leg receives 4429', async () => { + const observations = [] + const peer = new RelayLoadControlPeer(7, peerOptions(), (type, detail) => { + observations.push({ type, detail }) + }) + const control = fakeOpenSocket() + const phone = fakeHandshakeSocket() + const data = fakeHandshakeSocket() + let streamStarted + const streamStartedPromise = new Promise((resolve) => { streamStarted = resolve }) + phone._socket = { pause: () => undefined, resume: () => undefined } + control.send = (raw) => { + const message = JSON.parse(raw) + queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({ + type: 'invite-created', reqId: message.reqId, inviteToken: 'invite-token' + })))) + } + phone.send = (raw) => { + if (typeof raw === 'string' && raw.startsWith('{')) { + queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({ + type: 'conn-open', relayDeviceId: 'load-device-7-0', + connId: 'connection-7', connTicket: 'connection-ticket' + })))) + } + } + data.send = (raw) => { + if (typeof raw === 'string') { + queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) + } else { + streamStarted() + } + } + peer.socket = control + peer.generation = 11 + peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 9 } + control.on('message', (raw) => peer.onMessage(control, raw)) + peer.createClientSocket = () => openOnNextTurn(phone) + peer.createHostDataSocket = () => openOnNextTurn(data) + + await peer.openSplice({ + readerMode: 'wedged', + readerHoldMs: 10_001, + streamBytes: 300 * 1024, + frameBytes: 64 * 1024, + observeReaderPressure: async () => { + await streamStartedPromise + phone.close(1006) + data.close(4429, 'wedged relay link') + }, + readerDelay: async () => undefined + }) + + assert.equal(observations.filter(({ type }) => type === 'spliceWedged').length, 1) + assert.equal(observations.find(({ type }) => type === 'spliceWedged').detail.code, 4429) + await peer.shutdown() +}) + +test('requires one 4429 and rejects unrelated wedged close codes', () => { + assert.equal(relayLoadWedgedCloseAccepted([4429, 4429]), true) + assert.equal(relayLoadWedgedCloseAccepted([1006, 4429]), true) + assert.equal(relayLoadWedgedCloseAccepted([4429, 1006]), false) + assert.equal(relayLoadWedgedCloseAccepted([1006, 1006]), false) + assert.equal(relayLoadWedgedCloseAccepted([1000, 4429]), false) + assert.equal(relayLoadWedgedCloseAccepted([4429]), false) + assert.equal(relayLoadWedgedCloseAccepted([1006, 4429, 4429]), false) +}) + +test('streams reader frames with bounded send-side backpressure', async () => { + const peer = new RelayLoadControlPeer(9, peerOptions(), () => undefined) + let inFlight = 0 + let peakInFlight = 0 + let sentBytes = 0 + const socket = { + bufferedAmount: 0, + send(payload, callback) { + inFlight++ + peakInFlight = Math.max(peakInFlight, inFlight) + sentBytes += payload.byteLength + this.bufferedAmount = payload.byteLength + queueMicrotask(() => { + this.bufferedAmount = 0 + inFlight-- + callback() + }) + } + } + + const result = await peer.sendReaderStream( + socket, + 0, + 1024 * 1024, + 64 * 1024, + async () => undefined + ) + + assert.equal(result.bytes, 1024 * 1024) + assert.equal(sentBytes, 1024 * 1024) + assert.equal(peakInFlight, 1) + await peer.shutdown() +}) + +test('reports a bidirectional splice payload mismatch', async () => { + const observations = [] + const peer = new RelayLoadControlPeer(2, peerOptions(), (type) => observations.push(type)) + const control = fakeOpenSocket() + const phone = fakeHandshakeSocket() + const data = fakeHandshakeSocket() + control.send = (raw) => { + const message = JSON.parse(raw) + queueMicrotask(() => + control.emit( + 'message', + Buffer.from( + JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' }) + ) + ) + ) + } + phone.send = (raw) => { + if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') { + queueMicrotask(() => + control.emit( + 'message', + Buffer.from( + JSON.stringify({ + type: 'conn-open', + relayDeviceId: 'load-device-2-0', + connId: 'connection-2', + connTicket: 'ticket' + }) + ) + ) + ) + } + } + data.send = (raw) => { + if (typeof raw === 'string') { + queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) + } else { + queueMicrotask(() => phone.emit('message', Buffer.alloc(Buffer.from(raw).byteLength), true)) + } + } + peer.socket = control + peer.generation = 4 + peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 2 } + control.on('message', (raw) => peer.onMessage(control, raw)) + peer.createClientSocket = () => openOnNextTurn(phone) + peer.createHostDataSocket = () => openOnNextTurn(data) + + await assert.rejects(peer.openSplice(), /changed host-to-client splice payload/) + assert.equal(observations.includes('spliceFailed'), true) + await peer.shutdown() +}) + +test('shutdown closes both splice legs and waits for the in-flight splice', async () => { + const spliceOpened = deferred() + const observations = [] + const peer = new RelayLoadControlPeer(5, peerOptions(), (type, detail) => { + observations.push({ type, detail }) + if (type === 'spliceOpened') spliceOpened.resolve() + }) + const control = fakeOpenSocket() + const phone = fakeHandshakeSocket() + const data = fakeHandshakeSocket() + control.send = (raw) => { + const message = JSON.parse(raw) + queueMicrotask(() => + control.emit( + 'message', + Buffer.from( + JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' }) + ) + ) + ) + } + phone.send = (raw) => { + if (typeof raw === 'string' && raw.startsWith('{')) { + queueMicrotask(() => + control.emit( + 'message', + Buffer.from( + JSON.stringify({ + type: 'conn-open', + relayDeviceId: 'load-device-5-0', + connId: 'connection-5', + connTicket: 'ticket' + }) + ) + ) + ) + } else { + queueMicrotask(() => data.emit('message', Buffer.from(raw), false)) + } + } + data.send = (raw) => { + if (typeof raw === 'string') { + queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) + } else { + queueMicrotask(() => phone.emit('message', Buffer.from(raw), true)) + } + } + peer.socket = control + peer.generation = 8 + peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 3 } + control.on('message', (raw) => peer.onMessage(control, raw)) + peer.createClientSocket = () => openOnNextTurn(phone) + peer.createHostDataSocket = () => openOnNextTurn(data) + + const splice = peer.openSplice({ holdMs: 60_000 }) + await spliceOpened.promise + await Promise.all([splice, peer.shutdown()]) + + assert.equal(phone.readyState, phone.CLOSED) + assert.equal(data.readyState, data.CLOSED) + assert.equal(peer.inFlight.size, 0) + assert.equal(observations.at(-1).type, 'shutdown') + assert.equal(observations.at(-1).detail.activeSpliceSockets, 0) +}) diff --git a/cloud/dev/scripts/relay-load-director-capacity-gate.mjs b/cloud/dev/scripts/relay-load-director-capacity-gate.mjs new file mode 100644 index 00000000000..a1701d1a17e --- /dev/null +++ b/cloud/dev/scripts/relay-load-director-capacity-gate.mjs @@ -0,0 +1,147 @@ +import { setTimeout as delayDefault } from 'node:timers/promises' + +function integer(value) { + return Number.isSafeInteger(value) && value >= 0 ? value : undefined +} + +export function assertRelayLoadDirectorCapacityToken(config, now = Date.now, timeoutMs = 0) { + if (!config.adminToken || config.adminToken.length > 8_192) { + throw new Error('director capacity identity token is unavailable') + } + const origin = new URL(config.directorOrigin) + if (origin.protocol !== 'https:' || origin.origin !== config.directorOrigin) { + throw new Error('director capacity origin must be canonical HTTPS') + } + let claims + try { + const parts = config.adminToken.split('.') + if (parts.length !== 3) throw new Error('invalid token shape') + claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) + } catch { + throw new Error('director capacity identity token is invalid') + } + const expectedAudience = new URL('/v1/admin/drain', origin).toString() + const audiences = Array.isArray(claims.aud) ? claims.aud : [claims.aud] + const expiresAt = integer(claims.exp) + if ( + !audiences.includes(expectedAudience) || + typeof claims.email !== 'string' || + claims.email.length === 0 || + claims.email_verified !== true || + expiresAt === undefined || + expiresAt * 1_000 <= now() + timeoutMs + ) { + throw new Error('director capacity identity token is not bound to this proof') + } +} + +function matchingHeartbeat(status, config) { + const capacity = status?.connectionCapacity + const runtime = status?.runtime + const heartbeatAt = integer(runtime?.lastHeartbeatAt) + const matches = + status?.cellId === config.cellId && + status?.admissionState === 'general' && + runtime?.ready === true && + runtime?.heartbeatFresh === true && + capacity?.heartbeatFresh === true && + integer(capacity?.hardCap) === config.hardCap && + integer(capacity?.unobservedBound) === config.unobservedBound && + integer(capacity?.normalAdmissionPause) === config.requiredConnections && + integer(capacity?.observedConnections) === config.requiredConnections && + integer(capacity?.enforcedConnectionUnits) === config.requiredConnections && + integer(capacity?.inFlightConnections) === 0 && + integer(capacity?.reservedConnectionUnits) === 0 && + integer(capacity?.pendingControlReservations) === 0 && + heartbeatAt !== undefined + return matches ? heartbeatAt : undefined +} + +async function cellStatus(fetchImpl, config) { + const response = await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, { + method: 'POST', + headers: { + authorization: `Bearer ${config.adminToken}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, cellId: config.cellId }), + signal: AbortSignal.timeout(30_000) + }) + if (response.status === 401 || response.status === 403) { + throw new Error('director capacity identity was rejected') + } + if (!response.ok) { + await response.arrayBuffer().catch(() => undefined) + return undefined + } + const result = await response.json().catch(() => undefined) + if (!result?.status) throw new Error('director capacity status is invalid') + return result.status +} + +export async function waitForRelayLoadDirectorCapacity(config, overrides = {}) { + const fetchImpl = overrides.fetch ?? fetch + const delay = overrides.delay ?? delayDefault + const now = overrides.now ?? Date.now + const timeoutMs = overrides.timeoutMs ?? 120_000 + const pollMs = overrides.pollMs ?? 1_000 + assertRelayLoadDirectorCapacityToken(config, now, timeoutMs) + const deadline = now() + timeoutMs + let baselineHeartbeatAt = config.baselineHeartbeatAt + let previousHeartbeatAt + let matchingSamples = 0 + const requiredSamples = config.requiredSamples ?? 2 + for (;;) { + const status = await cellStatus(fetchImpl, config) + const currentHeartbeatAt = integer(status?.runtime?.lastHeartbeatAt) + if (baselineHeartbeatAt === undefined && currentHeartbeatAt !== undefined) { + baselineHeartbeatAt = currentHeartbeatAt + } + const heartbeatAt = status ? matchingHeartbeat(status, config) : undefined + if (heartbeatAt !== undefined && heartbeatAt > baselineHeartbeatAt) { + if (previousHeartbeatAt === undefined || heartbeatAt > previousHeartbeatAt) { + previousHeartbeatAt = heartbeatAt + matchingSamples++ + if (matchingSamples === requiredSamples) return { heartbeatAt } + } + } else { + previousHeartbeatAt = undefined + matchingSamples = 0 + } + if (now() >= deadline) throw new Error('director capacity did not converge after recovery') + await delay(pollMs) + } +} + +function matchingRequestUnits(status, config) { + return ( + status?.cellId === config.cellId && + status?.admissionState === 'general' && + status?.capacityRequests === config.capacityRequests && + status?.reservedRequests === config.expectedRequestUnits && + status?.activityRequestUnits === config.expectedRequestUnits && + status?.activityLeases === config.expectedActivityLeases && + status?.runtime?.observedRequests === config.expectedRequestUnits && + status?.runtime?.ready === true && + status?.runtime?.heartbeatFresh === true + ) +} + +export async function waitForRelayLoadRequestUnits(config, overrides = {}) { + const fetchImpl = overrides.fetch ?? fetch + const delay = overrides.delay ?? delayDefault + const now = overrides.now ?? Date.now + const timeoutMs = overrides.timeoutMs ?? config.timeoutMs ?? 120_000 + const pollMs = overrides.pollMs ?? 1_000 + const requiredSamples = overrides.requiredSamples ?? 2 + assertRelayLoadDirectorCapacityToken(config, now, timeoutMs) + const deadline = now() + timeoutMs + let matches = 0 + for (;;) { + const status = await cellStatus(fetchImpl, config) + matches = matchingRequestUnits(status, config) ? matches + 1 : 0 + if (matches === requiredSamples) return + if (now() >= deadline) throw new Error('Relay request-unit accounting did not converge') + await delay(pollMs) + } +} diff --git a/cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs b/cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs new file mode 100644 index 00000000000..fe0a2813b80 --- /dev/null +++ b/cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs @@ -0,0 +1,260 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + assertRelayLoadDirectorCapacityToken, + waitForRelayLoadDirectorCapacity, + waitForRelayLoadRequestUnits +} from './relay-load-director-capacity-gate.mjs' + +function token(claims) { + const encode = (value) => Buffer.from(JSON.stringify(value)).toString('base64url') + return `${encode({ alg: 'none' })}.${encode(claims)}.signature` +} + +const config = { + directorOrigin: 'https://relay-staging.example.com', + adminToken: token({ + aud: 'https://relay-staging.example.com/v1/admin/drain', + email: 'capacity@example.com', + email_verified: true, + exp: 4_000_000_000 + }), + cellId: 'staging-gce-c3', + hardCap: 1_000, + unobservedBound: 60, + requiredConnections: 840 +} + +function status(lastHeartbeatAt, overrides = {}) { + return { + cellId: config.cellId, + admissionState: 'general', + runtime: { ready: true, heartbeatFresh: true, lastHeartbeatAt, observedRequests: 0 }, + connectionCapacity: { + hardCap: 1_000, + unobservedBound: 60, + normalAdmissionPause: 840, + observedConnections: 840, + enforcedConnectionUnits: 840, + inFlightConnections: 0, + reservedConnectionUnits: 0, + pendingControlReservations: 0, + heartbeatFresh: true, + ...overrides + } + } +} + +function response(value, responseStatus = 200) { + return { + ok: responseStatus >= 200 && responseStatus < 300, + status: responseStatus, + json: async () => value, + arrayBuffer: async () => new ArrayBuffer(0) + } +} + +test('requires two advancing exact director capacity heartbeats', async () => { + const heartbeats = [101, 116, 131] + let calls = 0 + const result = await waitForRelayLoadDirectorCapacity(config, { + fetch: async () => response({ status: status(heartbeats[calls++]) }), + delay: async () => undefined + }) + assert.equal(calls, 3) + assert.deepEqual(result, { heartbeatAt: 131 }) +}) + +test('resets after a newer heartbeat undercounts recovered controls', async () => { + const samples = [ + status(101), + status(116), + status(131, { observedConnections: 839 }), + status(146), + status(161) + ] + let calls = 0 + await waitForRelayLoadDirectorCapacity(config, { + fetch: async () => response({ status: samples[calls++] }), + delay: async () => undefined + }) + assert.equal(calls, 5) +}) + +test('fails closed when exact advancing telemetry never converges', async () => { + let elapsed = 0 + await assert.rejects( + waitForRelayLoadDirectorCapacity(config, { + fetch: async () => response({ status: status(101) }), + delay: async (milliseconds) => { + elapsed += milliseconds + }, + now: () => elapsed, + timeoutMs: 2_000, + pollMs: 1_000 + }), + /did not converge/ + ) +}) + +test('rejects an unauthorized capacity identity without retrying', async () => { + let calls = 0 + await assert.rejects( + waitForRelayLoadDirectorCapacity(config, { + fetch: async () => { + calls++ + return response({}, 401) + }, + delay: async () => undefined + }), + /identity was rejected/ + ) + assert.equal(calls, 1) +}) + +test('binds the admin token to the canonical director audience', async () => { + await assert.rejects( + waitForRelayLoadDirectorCapacity( + { + ...config, + directorOrigin: 'https://relay-staging.example.com/path' + }, + { fetch: async () => response({ status: status(101) }), now: () => 0 } + ), + /canonical HTTPS/ + ) + await assert.rejects( + waitForRelayLoadDirectorCapacity( + { + ...config, + adminToken: token({ + aud: 'https://other.example.com/v1/admin/drain', + email: 'capacity@example.com', + email_verified: true, + exp: 4_000_000_000 + }) + }, + { fetch: async () => response({ status: status(101) }), now: () => 0 } + ), + /not bound/ + ) +}) + +test('rejects a missing or malformed admin token during startup preflight', () => { + assert.throws( + () => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: undefined }, () => 0), + /unavailable/ + ) + assert.throws( + () => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: 'not-a-jwt' }, () => 0), + /invalid/ + ) + assert.throws( + () => + assertRelayLoadDirectorCapacityToken( + { + ...config, + adminToken: token({ + aud: 'https://relay-staging.example.com/v1/admin/drain', + exp: 4_000_000_000 + }) + }, + () => 0 + ), + /not bound/ + ) +}) + +test('supports one newer exact post-probe heartbeat', async () => { + const heartbeats = [146, 161] + let calls = 0 + const result = await waitForRelayLoadDirectorCapacity( + { ...config, requiredSamples: 1 }, + { + fetch: async () => response({ status: status(heartbeats[calls++]) }), + delay: async () => undefined, + now: () => 0 + } + ) + assert.equal(calls, 2) + assert.deepEqual(result, { heartbeatAt: 161 }) +}) + +test('requires consecutive exact request-unit accounting samples', async () => { + const requestConfig = { + ...config, + capacityRequests: 6_000, + expectedRequestUnits: 6_000, + expectedActivityLeases: 6_000 + } + const samples = [5_999, 6_000, 6_000] + let calls = 0 + await waitForRelayLoadRequestUnits(requestConfig, { + fetch: async () => response({ + status: { + ...status(100), + runtime: { + ...status(100).runtime, + observedRequests: samples[calls] + }, + capacityRequests: 6_000, + reservedRequests: samples[calls], + activityRequestUnits: samples[calls], + activityLeases: samples[calls++] + } + }), + delay: async () => undefined, + now: () => 0 + }) + assert.equal(calls, 3) +}) + +test('requires the cell runtime to observe every request unit', async () => { + let elapsed = 0 + await assert.rejects(waitForRelayLoadRequestUnits({ + ...config, + capacityRequests: 6_000, + expectedRequestUnits: 6_000, + expectedActivityLeases: 6_000, + timeoutMs: 1_000 + }, { + fetch: async () => response({ + status: { + ...status(100), + runtime: { ...status(100).runtime, observedRequests: 5_999 }, + capacityRequests: 6_000, + reservedRequests: 6_000, + activityRequestUnits: 6_000, + activityLeases: 6_000 + } + }), + delay: async (milliseconds) => { elapsed += milliseconds }, + now: () => elapsed, + pollMs: 1_000 + }), /did not converge/) +}) + +test('fails closed when request-unit accounting does not clean up', async () => { + let elapsed = 0 + await assert.rejects(waitForRelayLoadRequestUnits({ + ...config, + capacityRequests: 6_000, + expectedRequestUnits: 0, + expectedActivityLeases: 0, + timeoutMs: 2_000 + }, { + fetch: async () => response({ + status: { + ...status(100), + runtime: { ...status(100).runtime, observedRequests: 1 }, + capacityRequests: 6_000, + reservedRequests: 1, + activityRequestUnits: 1, + activityLeases: 1 + } + }), + delay: async (milliseconds) => { elapsed += milliseconds }, + now: () => elapsed, + pollMs: 1_000 + }), /did not converge/) +}) diff --git a/cloud/dev/scripts/relay-load-model.mjs b/cloud/dev/scripts/relay-load-model.mjs new file mode 100644 index 00000000000..77de00422c6 --- /dev/null +++ b/cloud/dev/scripts/relay-load-model.mjs @@ -0,0 +1,76 @@ +const HEARTBEAT_INTERVAL_MS = 15_000 +const REFRESH_MIN_MS = 180_000 +const REFRESH_MAX_MS = 240_000 + +function mix32(value) { + let mixed = value >>> 0 + mixed = Math.imul(mixed ^ (mixed >>> 16), 0x21f0aaad) + mixed = Math.imul(mixed ^ (mixed >>> 15), 0x735a2d97) + return (mixed ^ (mixed >>> 15)) >>> 0 +} + +function fraction(seed, index, stream) { + return mix32(seed ^ Math.imul(index + 1, 0x9e3779b1) ^ stream) / 0x1_0000_0000 +} + +export function controlPhase(controlIndex, seed = 0x4f524341) { + const refreshIntervalMs = Math.round( + REFRESH_MIN_MS + fraction(seed, controlIndex, 2) * (REFRESH_MAX_MS - REFRESH_MIN_MS) + ) + return { + heartbeatOffsetMs: Math.floor(fraction(seed, controlIndex, 1) * HEARTBEAT_INTERVAL_MS), + refreshIntervalMs, + refreshOffsetMs: Math.floor(fraction(seed, controlIndex, 3) * refreshIntervalMs), + reconnectJitterMs: Math.floor(fraction(seed, controlIndex, 4) * 30_000) + } +} + +export function modeledRelayLoad(controlCount, durationMs = 15 * 60_000, seed) { + if (!Number.isInteger(controlCount) || controlCount < 1) throw new Error('controlCount must be positive') + const bins = Array.from({ length: Math.ceil(durationMs / 1000) }, () => ({ pings: 0, refreshes: 0 })) + for (let controlIndex = 0; controlIndex < controlCount; controlIndex++) { + const phase = controlPhase(controlIndex, seed) + for (let at = phase.heartbeatOffsetMs; at < durationMs; at += HEARTBEAT_INTERVAL_MS) { + bins[Math.floor(at / 1000)].pings++ + } + for (let at = phase.refreshOffsetMs; at < durationMs; at += phase.refreshIntervalMs) { + bins[Math.floor(at / 1000)].refreshes++ + } + } + const totals = bins.reduce( + (result, bin) => ({ + pings: result.pings + bin.pings, + refreshes: result.refreshes + bin.refreshes + }), + { pings: 0, refreshes: 0 } + ) + const durationSeconds = durationMs / 1000 + return { + controlCount, + durationMs, + expectedPingRate: controlCount / (HEARTBEAT_INTERVAL_MS / 1000), + expectedRefreshRate: controlCount / ((REFRESH_MIN_MS + REFRESH_MAX_MS) / 2 / 1000), + observedPingRate: totals.pings / durationSeconds, + observedRefreshRate: totals.refreshes / durationSeconds, + maxPingBurst: Math.max(...bins.map(({ pings }) => pings)), + maxRefreshBurst: Math.max(...bins.map(({ refreshes }) => refreshes)) + } +} + +export function assertSpreadModel(model) { + const pingTolerance = model.expectedPingRate * 0.03 + 1 + const refreshTolerance = model.expectedRefreshRate * 0.08 + 1 + if (Math.abs(model.observedPingRate - model.expectedPingRate) > pingTolerance) { + throw new Error('modeled heartbeat rate diverged from the 15-second contract') + } + if (Math.abs(model.observedRefreshRate - model.expectedRefreshRate) > refreshTolerance) { + throw new Error('modeled token refresh rate diverged from the 180-240 second contract') + } + if (model.maxPingBurst > model.expectedPingRate * 1.3 + 5) { + throw new Error('heartbeat phase spreading produced a reconnect cliff') + } + // One-second bins have Poisson-sized tails even with uniform phase spreading. + if (model.maxRefreshBurst > model.expectedRefreshRate * 1.75 + 5) { + throw new Error('refresh phase spreading produced an auth herd') + } +} diff --git a/cloud/dev/scripts/relay-load-model.test.mjs b/cloud/dev/scripts/relay-load-model.test.mjs new file mode 100644 index 00000000000..e38548efe3b --- /dev/null +++ b/cloud/dev/scripts/relay-load-model.test.mjs @@ -0,0 +1,25 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { assertSpreadModel, controlPhase, modeledRelayLoad } from './relay-load-model.mjs' + +test('phases are deterministic, bounded, and separated by stream', () => { + assert.deepEqual(controlPhase(42), controlPhase(42)) + assert.notDeepEqual(controlPhase(42), controlPhase(43)) + const phase = controlPhase(42) + assert.ok(phase.heartbeatOffsetMs >= 0 && phase.heartbeatOffsetMs < 15_000) + assert.ok(phase.refreshIntervalMs >= 180_000 && phase.refreshIntervalMs <= 240_000) + assert.ok(phase.refreshOffsetMs >= 0 && phase.refreshOffsetMs < phase.refreshIntervalMs) + assert.ok(phase.reconnectJitterMs >= 0 && phase.reconnectJitterMs < 30_000) +}) + +for (const [controls, expectedPings, expectedRefreshes] of [ + [4_000, 267, 19], + [10_000, 667, 48] +]) { + test(`${controls} modeled controls spread heartbeat and token refresh load`, () => { + const model = modeledRelayLoad(controls) + assert.equal(Math.round(model.expectedPingRate), expectedPings) + assert.equal(Math.round(model.expectedRefreshRate), expectedRefreshes) + assert.doesNotThrow(() => assertSpreadModel(model)) + }) +} diff --git a/cloud/dev/scripts/relay-load-phase-barrier.mjs b/cloud/dev/scripts/relay-load-phase-barrier.mjs new file mode 100644 index 00000000000..03e0e800074 --- /dev/null +++ b/cloud/dev/scripts/relay-load-phase-barrier.mjs @@ -0,0 +1,37 @@ +import { access, mkdir, open } from 'node:fs/promises' +import { join } from 'node:path' +import { setTimeout as delayDefault } from 'node:timers/promises' + +export async function waitForRelayLoadPhaseBarrier(config, overrides = {}) { + const delay = overrides.delay ?? delayDefault + const now = overrides.now ?? Date.now + const timeoutMs = overrides.timeoutMs ?? config.timeoutMs + if ( + typeof config.directory !== 'string' || config.directory.length === 0 || + !Number.isSafeInteger(config.shardCount) || config.shardCount < 2 || + !Number.isSafeInteger(config.shardIndex) || config.shardIndex < 0 || + config.shardIndex >= config.shardCount || + !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 + ) throw new Error('invalid Relay load phase barrier') + + await mkdir(config.directory, { recursive: true }) + const marker = join(config.directory, `${config.shardIndex}.ready`) + const handle = await open(marker, 'wx') + await handle.close() + const deadline = now() + timeoutMs + for (;;) { + const ready = await Promise.all( + Array.from({ length: config.shardCount }, async (_, index) => { + try { + await access(join(config.directory, `${index}.ready`)) + return true + } catch { + return false + } + }) + ) + if (ready.every(Boolean)) return + if (now() >= deadline) throw new Error('Relay load phase barrier timed out') + await delay(100) + } +} diff --git a/cloud/dev/scripts/relay-load-phase-barrier.test.mjs b/cloud/dev/scripts/relay-load-phase-barrier.test.mjs new file mode 100644 index 00000000000..2fc16d3650a --- /dev/null +++ b/cloud/dev/scripts/relay-load-phase-barrier.test.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs' + +const loadHarness = await readFile(new URL('./load-relay-controls.mjs', import.meta.url), 'utf8') + +test('releases every shard only after all readiness markers exist', async () => { + const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-')) + try { + let firstResolved = false + const first = waitForRelayLoadPhaseBarrier({ + directory, shardCount: 2, shardIndex: 0, timeoutMs: 1_000 + }).then(() => { firstResolved = true }) + await new Promise((resolve) => setTimeout(resolve, 20)) + assert.equal(firstResolved, false) + await Promise.all([ + first, + waitForRelayLoadPhaseBarrier({ + directory, shardCount: 2, shardIndex: 1, timeoutMs: 1_000 + }) + ]) + assert.equal(firstResolved, true) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test('fails closed on a duplicate shard or incomplete barrier', async () => { + const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-')) + try { + const nowValues = [0, 2] + await assert.rejects( + waitForRelayLoadPhaseBarrier( + { directory, shardCount: 2, shardIndex: 0, timeoutMs: 1 }, + { now: () => nowValues.shift() ?? 2, delay: async () => undefined } + ), + /timed out/ + ) + await assert.rejects( + waitForRelayLoadPhaseBarrier({ + directory, shardCount: 2, shardIndex: 0, timeoutMs: 1 + }), + /EEXIST/ + ) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test('synchronizes splice ramps after every shard finishes reader baselines', () => { + assert.match( + loadHarness, + /createRelayLoadReaderEvidence[\s\S]*?phaseBarrierDir\}-splices[\s\S]*?splicePromises/ + ) +}) + +test('budgets both shard barriers and the splice ramp in token lifetime', () => { + assert.match( + loadHarness, + /phaseBarrierDir \? 2 \* config\.phaseBarrierTimeoutMs : 0[\s\S]*?config\.spliceRampMs/ + ) +}) diff --git a/cloud/dev/scripts/relay-load-placement-boundary.mjs b/cloud/dev/scripts/relay-load-placement-boundary.mjs new file mode 100644 index 00000000000..84f19b1dc80 --- /dev/null +++ b/cloud/dev/scripts/relay-load-placement-boundary.mjs @@ -0,0 +1,29 @@ +export async function proveRelayLoadPlacementBoundary({ peer, failureReason }) { + let connected = false + try { + await peer.connect() + connected = true + } catch (error) { + const reason = failureReason(error) + if (reason !== 'assignment_capacity_exhausted') { + throw new Error(`placement overflow was not rejected: ${reason}`) + } + return reason + } finally { + await peer.shutdown() + } + if (connected) throw new Error('placement overflow unexpectedly connected') +} + +export async function proveRelayLoadRegionalFallback({ peer, blockedOrigin }) { + try { + await peer.connect() + const assignedOrigin = peer.assignedCellUrl() + if (!assignedOrigin || assignedOrigin === blockedOrigin) { + throw new Error('regional fallback did not leave the full preferred cell') + } + return true + } finally { + await peer.shutdown() + } +} diff --git a/cloud/dev/scripts/relay-load-placement-boundary.test.mjs b/cloud/dev/scripts/relay-load-placement-boundary.test.mjs new file mode 100644 index 00000000000..89dd338816f --- /dev/null +++ b/cloud/dev/scripts/relay-load-placement-boundary.test.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + proveRelayLoadPlacementBoundary, + proveRelayLoadRegionalFallback +} from './relay-load-placement-boundary.mjs' + +function peer(connect) { + return { connect, shutdown: async () => undefined } +} + +test('requires the next fresh placement to receive HTTP 503', async () => { + assert.equal( + await proveRelayLoadPlacementBoundary({ + peer: peer(async () => { + throw new Error('relay assignment failed: 503 relay_connection_headroom_exhausted') + }), + failureReason: (error) => + error.message === 'relay assignment failed: 503 relay_connection_headroom_exhausted' + ? 'assignment_capacity_exhausted' + : 'unknown' + }), + 'assignment_capacity_exhausted' + ) + await assert.rejects( + proveRelayLoadPlacementBoundary({ + peer: peer(async () => undefined), + failureReason: () => 'unknown' + }), + /placement overflow unexpectedly connected/ + ) +}) + +test('requires a preferred-region fallback to leave the full cell', async () => { + let shutdowns = 0 + assert.equal(await proveRelayLoadRegionalFallback({ + peer: { + connect: async () => undefined, + assignedCellUrl: () => 'https://c3.relay-staging.onorca.dev', + shutdown: async () => { shutdowns++ } + }, + blockedOrigin: 'https://c4.relay-staging.onorca.dev' + }), true) + assert.equal(shutdowns, 1) + await assert.rejects(proveRelayLoadRegionalFallback({ + peer: { + connect: async () => undefined, + assignedCellUrl: () => 'https://c4.relay-staging.onorca.dev', + shutdown: async () => undefined + }, + blockedOrigin: 'https://c4.relay-staging.onorca.dev' + }), /did not leave/) +}) diff --git a/cloud/dev/scripts/relay-load-profile.mjs b/cloud/dev/scripts/relay-load-profile.mjs new file mode 100644 index 00000000000..0a0552fc029 --- /dev/null +++ b/cloud/dev/scripts/relay-load-profile.mjs @@ -0,0 +1,404 @@ +export const RELAY_CONTROL_LEASE_HORIZON_SECONDS = 105 +export const RELAY_LOAD_SPLICE_HIGH_WATER_BYTES = 256 * 1024 +export const RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS = 10_000 +export const RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES = 16 +export const RELAY_LOAD_MAX_AGGREGATE_READER_BYTES = 64 * 1024 * 1024 + +const DEFAULT_SLOW_READER_STREAM_BYTES = 1024 * 1024 +const DEFAULT_WEDGED_READER_STREAM_BYTES = 8 * 1024 * 1024 +const DEFAULT_READER_FRAME_BYTES = 64 * 1024 + +export function parseRelayLoadArguments(argv) { + const values = new Map() + const flags = new Set() + for (let index = 0; index < argv.length; index++) { + const argument = argv[index] + if (argument === '--') continue + if ( + [ + '--allow-partial', + '--allow-planned-transition-retries', + '--skip-rebind-overflow-check' + ].includes(argument) + ) { + flags.add(argument) + continue + } + if (!argument.startsWith('--') || index + 1 >= argv.length) { + throw new Error(`invalid argument: ${argument}`) + } + values.set(argument, argv[++index]) + } + const integer = (name, fallback) => { + const parsed = Number(values.get(name) ?? fallback) + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a nonnegative integer`) + } + return parsed + } + const targetOrigin = values.get('--target-origin')?.replace(/\/$/, '') + const directorOrigin = values.get('--director-origin')?.replace(/\/$/, '') + const authOrigin = values.get('--auth-origin')?.replace(/\/$/, '') + if ((!targetOrigin && !directorOrigin) || (targetOrigin && directorOrigin) || !authOrigin) { + throw new Error('provide --auth-origin and exactly one target or director origin') + } + const controls = integer('--controls', 100) + const maxRampConnectionFailures = integer('--max-ramp-connection-failures', 0) + const maxUnexpectedCloses = integer('--max-unexpected-closes', 0) + const rebindProbes = integer('--rebind-probes', 0) + const placementOverflowProbes = integer('--placement-overflow-probes', 0) + const regionalFallbackProbes = integer('--regional-fallback-probes', 0) + const regionBehaviorProbes = integer('--region-behavior-probes', 0) + const requestUnitInvites = integer('--request-unit-invites', 0) + const requestUnitInvitesPerSecond = integer('--request-unit-invites-per-second', 0) + const requestUnitPrincipalCount = integer('--request-unit-principals', 0) + const relayAsiaLoadPrincipalCount = integer('--relay-asia-load-principals', 0) + const requestUnitOverflowProbes = integer('--request-unit-overflow-probes', 0) + const requestUnitCapacity = values.has('--request-unit-capacity') + ? integer('--request-unit-capacity', 0) + : undefined + const requestUnitCleanupTimeoutMs = integer( + '--request-unit-cleanup-timeout-seconds', + 0 + ) * 1000 + const rebindHoldMs = integer('--rebind-hold-ms', 4_000) + const rebindDelayMs = integer('--rebind-delay-seconds', 0) * 1000 + const capacityCellId = values.get('--capacity-cell-id') + const capacityCellOrigin = values.get('--capacity-cell-origin')?.replace(/\/$/, '') + const capacityHardCap = values.has('--capacity-hard-cap') + ? integer('--capacity-hard-cap', 0) + : undefined + const capacityUnobservedBound = values.has('--capacity-unobserved-bound') + ? integer('--capacity-unobserved-bound', 0) + : undefined + const shardCount = integer('--shard-count', 1) + const shardIndex = integer('--shard-index', 0) + const durationMs = integer('--duration-seconds', 900) * 1000 + const spliceHoldMs = integer( + '--splice-hold-seconds', + values.get('--duration-seconds') ?? 900 + ) * 1000 + const splices = integer('--splices', 0) + const spliceRampMs = integer('--splice-ramp-seconds', 0) * 1000 + const slowReaderSplices = integer('--slow-reader-splices', 0) + const wedgedReaderSplices = integer('--wedged-reader-splices', 0) + const splicePayloadBytes = integer('--splice-payload-bytes', 1_024) + const slowReaderStreamBytes = integer( + '--slow-reader-stream-bytes', + DEFAULT_SLOW_READER_STREAM_BYTES + ) + const wedgedReaderStreamBytes = integer( + '--wedged-reader-stream-bytes', + DEFAULT_WEDGED_READER_STREAM_BYTES + ) + const readerFrameBytes = integer('--reader-frame-bytes', DEFAULT_READER_FRAME_BYTES) + const slowReaderHoldMs = integer('--slow-reader-hold-ms', 2_000) + const wedgedReaderHoldMs = integer('--wedged-reader-hold-ms', 12_000) + const maxGeneratorRssGrowthMiB = integer('--max-generator-rss-growth-mib', 512) + const requiredLeaseHorizons = integer('--required-lease-horizons', 0) + const aggregateControls = values.has('--aggregate-controls') + ? integer('--aggregate-controls', 0) + : controls + const aggregateSplices = values.has('--aggregate-splices') + ? integer('--aggregate-splices', 0) + : splices + const aggregateRequestUnitInvites = values.has('--aggregate-request-unit-invites') + ? integer('--aggregate-request-unit-invites', 0) + : requestUnitInvites + const phaseBarrierDir = values.get('--phase-barrier-dir') + const phaseBarrierTimeoutMs = integer('--phase-barrier-timeout-seconds', 180) * 1000 + if (controls < 1 || controls > 10_000) throw new Error('--controls must be between 1 and 10000') + if (rebindProbes > controls) throw new Error('--rebind-probes cannot exceed --controls') + if (splices > controls) throw new Error('--splices cannot exceed --controls') + if (shardCount > 1 && capacityHardCap !== undefined) { + if (!values.has('--aggregate-controls') || !values.has('--aggregate-splices')) { + throw new Error('capacity-bound sharding requires explicit aggregate controls and splices') + } + if (aggregateControls !== controls * shardCount || aggregateSplices !== splices * shardCount) { + throw new Error('aggregate controls and splices must match every equal-sized shard') + } + } else if (aggregateControls !== controls || aggregateSplices !== splices) { + throw new Error('aggregate controls and splices require matching sharded local counts') + } + if ( + capacityHardCap !== undefined && + aggregateControls + 2 * aggregateSplices > capacityHardCap - 100 + ) { + throw new Error('controls plus splice connection units exceed ordinary cell admission') + } + if (slowReaderSplices + wedgedReaderSplices > splices) { + throw new Error('reader splice counts cannot exceed --splices') + } + if (splices > 0 && (spliceHoldMs < 1_000 || spliceHoldMs > durationMs)) { + throw new Error('--splice-hold-seconds must be between 1 and the steady duration') + } + if (slowReaderSplices + wedgedReaderSplices > 0 && !directorOrigin) { + throw new Error('reader evidence requires --director-origin') + } + if (splicePayloadBytes < 1 || splicePayloadBytes > 1_048_576) { + throw new Error('--splice-payload-bytes must be between 1 and 1048576') + } + if ( + slowReaderSplices > 0 && + slowReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES + ) { + throw new Error('--slow-reader-stream-bytes must exceed the 256 KiB splice high-water mark') + } + if ( + wedgedReaderSplices > 0 && + wedgedReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES + ) { + throw new Error('--wedged-reader-stream-bytes must exceed the 256 KiB splice high-water mark') + } + const localReaderSplices = slowReaderSplices + wedgedReaderSplices + const localReaderBytes = slowReaderSplices * slowReaderStreamBytes + + wedgedReaderSplices * wedgedReaderStreamBytes + const aggregateReaderSplices = values.has('--aggregate-reader-splices') + ? integer('--aggregate-reader-splices', 0) + : localReaderSplices * shardCount + const aggregateReaderBytes = values.has('--aggregate-reader-bytes') + ? integer('--aggregate-reader-bytes', 0) + : localReaderBytes * shardCount + if ( + aggregateReaderSplices < localReaderSplices || + aggregateReaderBytes < localReaderBytes || + (shardCount === 1 && + (aggregateReaderSplices !== localReaderSplices || aggregateReaderBytes !== localReaderBytes)) + ) { + throw new Error('aggregate reader bounds do not cover the local shard') + } + if (aggregateReaderSplices > RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES) { + throw new Error('aggregate reader splice count exceeds the reviewed bound') + } + if (aggregateReaderBytes > RELAY_LOAD_MAX_AGGREGATE_READER_BYTES) { + throw new Error('aggregate reader stream bytes exceed the reviewed bound') + } + if (readerFrameBytes < 1 || readerFrameBytes > 1_048_576) { + throw new Error('--reader-frame-bytes must be between 1 and 1048576') + } + if (slowReaderSplices > 0 && slowReaderHoldMs >= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) { + throw new Error('--slow-reader-hold-ms must stay below the wedged timeout') + } + if (wedgedReaderSplices > 0 && wedgedReaderHoldMs <= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) { + throw new Error('--wedged-reader-hold-ms must exceed the wedged timeout') + } + if (wedgedReaderHoldMs > 30_000) { + throw new Error('--wedged-reader-hold-ms cannot exceed 30000') + } + if (maxGeneratorRssGrowthMiB < 1) { + throw new Error('--max-generator-rss-growth-mib must be positive') + } + if (placementOverflowProbes > 1) { + throw new Error('--placement-overflow-probes must be zero or one') + } + if (regionalFallbackProbes > 1) { + throw new Error('--regional-fallback-probes must be zero or one') + } + if (regionBehaviorProbes > 1 || requestUnitOverflowProbes > 1) { + throw new Error('regional behavior and request-unit overflow probes must be zero or one') + } + if (placementOverflowProbes > 0 && shardCount > 1) { + throw new Error('placement overflow proof requires one coordinated generator') + } + if ( + placementOverflowProbes > 0 && + (!directorOrigin || + !capacityCellId || + capacityHardCap === undefined || + capacityUnobservedBound === undefined) + ) { + throw new Error('placement overflow requires exact capacity cell, hard cap, and bound') + } + if (regionalFallbackProbes > 0 && ( + !directorOrigin || !capacityCellId || !capacityCellOrigin || + capacityHardCap === undefined || capacityUnobservedBound === undefined || + (shardCount > 1 && shardIndex !== 0) + )) throw new Error('regional fallback requires the coordinating capacity shard') + if (regionBehaviorProbes > 0 && ( + !directorOrigin || !capacityCellOrigin || preferredRegionValue(values) !== 'asia-east2' || + (shardCount > 1 && shardIndex !== 0) + )) throw new Error('regional behavior proof requires the coordinating Asia shard') + if (phaseBarrierDir && shardCount < 2) { + throw new Error('phase barrier requires multiple shards') + } + if (phaseBarrierTimeoutMs < 1_000) { + throw new Error('phase barrier timeout must be at least one second') + } + if (requestUnitInvites > 0) { + if ( + !directorOrigin || !capacityCellId || requestUnitCapacity === undefined || + requestUnitCapacity < 1 || requestUnitInvitesPerSecond < 1 || + requestUnitInvitesPerSecond > 20 || requestUnitPrincipalCount < 1 || + requestUnitPrincipalCount > 32 || requestUnitInvites > requestUnitPrincipalCount * 30 || + aggregateSplices !== 0 || !phaseBarrierDir || + aggregateRequestUnitInvites !== requestUnitInvites * shardCount || + aggregateControls + aggregateRequestUnitInvites !== requestUnitCapacity + ) throw new Error('request-unit proof does not reach the exact reviewed capacity') + } else if ( + requestUnitCapacity !== undefined || requestUnitInvitesPerSecond !== 0 || + requestUnitPrincipalCount !== 0 || + requestUnitOverflowProbes > 0 || + requestUnitCleanupTimeoutMs > 0 || aggregateRequestUnitInvites !== 0 + ) throw new Error('request-unit proof options require invite offers') + if ( + requestUnitOverflowProbes > 0 && + (shardIndex !== 0 || requestUnitCleanupTimeoutMs < 600_000) + ) throw new Error('request-unit overflow requires the cleanup-owning coordinator') + if (requestUnitCleanupTimeoutMs > 0 && requestUnitOverflowProbes !== 1) { + throw new Error('request-unit cleanup requires the overflow proof') + } + if ( + relayAsiaLoadPrincipalCount > 32 || + (relayAsiaLoadPrincipalCount > 0 && + (!directorOrigin || preferredRegionValue(values) !== 'asia-east2')) + ) throw new Error('Relay Asia load principals require a regional director proof') + if (capacityCellOrigin) { + const origin = new URL(capacityCellOrigin) + if (origin.protocol !== 'https:' || origin.origin !== capacityCellOrigin) { + throw new Error('--capacity-cell-origin must be canonical HTTPS') + } + } + if (shardCount < 1 || shardIndex >= shardCount) throw new Error('invalid shard index/count') + const minimumDurationMs = requiredLeaseHorizons * RELAY_CONTROL_LEASE_HORIZON_SECONDS * 1000 + if (durationMs < minimumDurationMs) { + throw new Error(`--duration-seconds must cover ${requiredLeaseHorizons} lease horizons`) + } + return { + targetOrigin, + directorOrigin, + authOrigin, + preferredRegion: preferredRegionValue(values), + controls, + maxRampConnectionFailures, + maxUnexpectedCloses, + rebindProbes, + placementOverflowProbes, + regionalFallbackProbes, + regionBehaviorProbes, + requestUnitInvites, + requestUnitInvitesPerSecond, + requestUnitPrincipalCount, + relayAsiaLoadPrincipalCount, + requestUnitOverflowProbes, + requestUnitCapacity, + requestUnitCleanupTimeoutMs, + rebindHoldMs, + rebindDelayMs, + capacityCellId, + capacityCellOrigin, + capacityHardCap, + capacityUnobservedBound, + durationMs, + spliceHoldMs, + rampMs: integer('--ramp-seconds', 60) * 1000, + rampStartDelayMs: integer('--ramp-start-delay-ms', 0), + reconnectMaxMs: integer('--reconnect-max-seconds', 30) * 1000, + shardCount, + shardIndex, + aggregateControls, + aggregateSplices, + aggregateRequestUnitInvites, + phaseBarrierDir, + phaseBarrierTimeoutMs, + aggregateReaderSplices, + aggregateReaderBytes, + signingKeyFile: values.get('--signing-key-file'), + splices, + spliceRampMs, + splicePayloadBytes, + slowReaderSplices, + wedgedReaderSplices, + slowReaderStreamBytes, + wedgedReaderStreamBytes, + readerFrameBytes, + slowReaderHoldMs, + wedgedReaderHoldMs, + maxGeneratorRssGrowthMiB, + requiredLeaseHorizons, + allowPartial: flags.has('--allow-partial'), + allowPlannedTransitionRetries: flags.has('--allow-planned-transition-retries'), + requireRebindOverflow: !flags.has('--skip-rebind-overflow-check') + } +} + +function preferredRegionValue(values) { + return values.get('--preferred-region') +} + +export function relayLoadSpliceIndexes(config) { + return Array.from( + { length: config.splices }, + (_, localIndex) => localIndex * config.shardCount + config.shardIndex + ) +} + +export function relayLoadSpliceStartDelayMs(config, localIndex) { + if (!Number.isSafeInteger(localIndex) || localIndex < 0 || localIndex >= config.splices) { + throw new Error('invalid local splice index') + } + const totalSplices = config.splices * config.shardCount + if (totalSplices <= 1) return 0 + const globalOrdinal = localIndex * config.shardCount + config.shardIndex + return Math.floor(globalOrdinal * config.spliceRampMs / (totalSplices - 1)) +} + +export function relayLoadPrincipalIndex(peerIndex, shardCount, principalCount) { + if ( + !Number.isSafeInteger(peerIndex) || peerIndex < 0 || + !Number.isSafeInteger(shardCount) || shardCount < 1 || + !Number.isSafeInteger(principalCount) || principalCount < 1 + ) throw new Error('invalid Relay load principal mapping') + return Math.floor(peerIndex / shardCount) % principalCount +} + +export function relayLoadSpliceProfile(config, spliceIndex) { + if (spliceIndex < config.wedgedReaderSplices) { + return { + readerMode: 'wedged', + readerHoldMs: config.wedgedReaderHoldMs, + streamBytes: config.wedgedReaderStreamBytes, + frameBytes: config.readerFrameBytes + } + } + if (spliceIndex < config.wedgedReaderSplices + config.slowReaderSplices) { + return { + readerMode: 'slow', + readerHoldMs: config.slowReaderHoldMs, + streamBytes: config.slowReaderStreamBytes, + frameBytes: config.readerFrameBytes + } + } + return { + readerMode: 'normal', + readerHoldMs: 0, + streamBytes: config.splicePayloadBytes, + frameBytes: config.splicePayloadBytes + } +} + +export function relayLoadReaderEvidenceError(result, config) { + const readerSplices = config.slowReaderSplices + config.wedgedReaderSplices + if (readerSplices > 0 && result.generatorRssGrowthMiB > config.maxGeneratorRssGrowthMiB) { + return 'load generator exceeded its RSS growth budget' + } + if (result.slowReaderSplicesCompleted !== config.slowReaderSplices) { + return 'slow-reader streams did not all complete' + } + if (result.wedgedReaderSplicesClosed !== config.wedgedReaderSplices) { + return 'wedged-reader streams did not all close at the relay limit' + } + if ( + readerSplices > 0 && + (result.readerQueueEvidence.length === 0 || + result.readerQueueEvidence.some(({ increaseBytes }) => increaseBytes < 1)) + ) { + return 'reader streams produced no causal Relay queued-byte evidence' + } + if ( + config.wedgedReaderSplices > 0 && + result.readerClosesByCode['4429'] !== config.wedgedReaderSplices + ) { + return 'wedged-reader streams did not close with 4429' + } + return undefined +} diff --git a/cloud/dev/scripts/relay-load-profile.test.mjs b/cloud/dev/scripts/relay-load-profile.test.mjs new file mode 100644 index 00000000000..40724f9c34e --- /dev/null +++ b/cloud/dev/scripts/relay-load-profile.test.mjs @@ -0,0 +1,388 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + parseRelayLoadArguments, + relayLoadPrincipalIndex, + relayLoadReaderEvidenceError, + relayLoadSpliceIndexes, + relayLoadSpliceProfile, + relayLoadSpliceStartDelayMs +} from './relay-load-profile.mjs' + +const required = ['--auth-origin', 'https://auth.test', '--director-origin', 'https://relay.test'] + +test('accepts the 2840-control two-lease-horizon Asia proof profile', () => { + const config = parseRelayLoadArguments([ + ...required, + '--controls', + '2840', + '--duration-seconds', + '210', + '--required-lease-horizons', + '2', + '--preferred-region', + 'asia-east2', + '--relay-asia-load-principals', + '32', + '--capacity-hard-cap', + '3000' + ]) + + assert.equal(config.controls, 2840) + assert.equal(config.durationMs, 210_000) + assert.equal(config.requiredLeaseHorizons, 2) + assert.equal(config.preferredRegion, 'asia-east2') + assert.equal(config.relayAsiaLoadPrincipalCount, 32) + assert.equal(config.splices, 0) +}) + +test('binds synthetic Relay principals to a bounded Asia director proof', () => { + assert.throws(() => parseRelayLoadArguments([ + ...required, '--relay-asia-load-principals', '1' + ]), /require a regional director proof/) + assert.throws(() => parseRelayLoadArguments([ + ...required, '--preferred-region', 'asia-east2', + '--relay-asia-load-principals', '33' + ]), /require a regional director proof/) +}) + +test('rejects a mixed profile beyond the ordinary 2900-unit boundary', () => { + assert.throws( + () => parseRelayLoadArguments([ + ...required, + '--controls', '2840', + '--splices', '31', + '--capacity-hard-cap', '3000' + ]), + /exceed ordinary cell admission/ + ) + expectMixedProfile(parseRelayLoadArguments([ + ...required, + '--controls', '2600', + '--splices', '120', + '--capacity-hard-cap', '3000' + ])) +}) + +test('requires reviewed aggregate totals for capacity-bound shards', () => { + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--controls', '2840', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0' + ]), + /requires explicit aggregate controls and splices/ + ) + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--controls', '2840', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0', + '--aggregate-controls', '2840', '--aggregate-splices', '0' + ]), + /must match every equal-sized shard/ + ) + const config = parseRelayLoadArguments([ + ...required, '--controls', '710', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0', + '--aggregate-controls', '2840', '--aggregate-splices', '0' + ]) + assert.equal(config.aggregateControls, 2840) + assert.equal(config.aggregateSplices, 0) +}) + +test('allows one coordinating shard to prove regional fallback', () => { + const config = parseRelayLoadArguments([ + ...required, '--controls', '710', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0', + '--aggregate-controls', '2840', '--aggregate-splices', '0', + '--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4', + '--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev', + '--capacity-unobserved-bound', '60', '--rebind-probes', '160' + ]) + assert.equal(config.regionalFallbackProbes, 1) + assert.equal(config.capacityCellOrigin, 'https://c4.relay-staging.onorca.dev') + assert.throws(() => parseRelayLoadArguments([ + ...required, '--controls', '710', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '1', + '--aggregate-controls', '2840', '--aggregate-splices', '0', + '--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4', + '--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev', + '--capacity-unobserved-bound', '60' + ]), /coordinating capacity shard/) +}) + +test('accepts the exact sharded request-unit and region behavior proof', () => { + const config = parseRelayLoadArguments([ + ...required, '--preferred-region', 'asia-east2', + '--controls', '710', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0', + '--aggregate-controls', '2840', '--aggregate-splices', '0', + '--capacity-cell-id', 'staging-gce-c4', + '--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev', + '--request-unit-invites', '790', '--request-unit-invites-per-second', '2', + '--request-unit-principals', '32', + '--aggregate-request-unit-invites', '3160', + '--request-unit-capacity', '6000', '--request-unit-overflow-probes', '1', + '--request-unit-cleanup-timeout-seconds', '630', + '--region-behavior-probes', '1', '--phase-barrier-dir', '/tmp/load-barrier' + ]) + assert.equal(config.aggregateRequestUnitInvites, 3_160) + assert.equal(config.requestUnitCapacity, 6_000) + assert.equal(config.requestUnitPrincipalCount, 32) + assert.equal(config.requestUnitCleanupTimeoutMs, 630_000) + assert.equal(config.regionBehaviorProbes, 1) + assert.equal(config.phaseBarrierDir, '/tmp/load-barrier') + + assert.throws(() => parseRelayLoadArguments([ + ...required, '--controls', '710', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0', + '--aggregate-controls', '2840', '--aggregate-splices', '0', + '--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '789', + '--request-unit-invites-per-second', '2', + '--request-unit-principals', '32', + '--aggregate-request-unit-invites', '3156', '--request-unit-capacity', '6000', + '--phase-barrier-dir', '/tmp/load-barrier' + ]), /does not reach the exact reviewed capacity/) + + assert.throws(() => parseRelayLoadArguments([ + ...required, '--controls', '710', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0', + '--aggregate-controls', '2840', '--aggregate-splices', '0', + '--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '790', + '--request-unit-invites-per-second', '2', '--request-unit-principals', '26', + '--aggregate-request-unit-invites', '3160', '--request-unit-capacity', '6000', + '--phase-barrier-dir', '/tmp/load-barrier' + ]), /does not reach the exact reviewed capacity/) +}) + +function expectMixedProfile(config) { + assert.equal(config.controls + 2 * config.splices, 2840) +} + +test('rejects a run shorter than its required lease horizons', () => { + assert.throws( + () => + parseRelayLoadArguments([ + ...required, + '--duration-seconds', + '209', + '--required-lease-horizons', + '2' + ]), + /must cover 2 lease horizons/ + ) +}) + +test('bounds an explicit splice hold within the steady window', () => { + const config = parseRelayLoadArguments([ + ...required, + '--controls', '1', + '--splices', '1', + '--duration-seconds', '300', + '--splice-hold-seconds', '60' + ]) + assert.equal(config.durationMs, 300_000) + assert.equal(config.spliceHoldMs, 60_000) + assert.throws(() => parseRelayLoadArguments([ + ...required, + '--controls', '1', + '--splices', '1', + '--duration-seconds', '300', + '--splice-hold-seconds', '301' + ]), /between 1 and the steady duration/) +}) + +test('maps splice ownership deterministically within a shard', () => { + const config = parseRelayLoadArguments([ + ...required, + '--controls', + '4', + '--splices', + '3', + '--shard-count', + '4', + '--shard-index', + '2' + ]) + + assert.deepEqual(relayLoadSpliceIndexes(config), [2, 6, 10]) +}) + +test('staggered shards form one deterministic splice ramp', () => { + const config = parseRelayLoadArguments([ + ...required, + '--controls', '4', + '--splices', '3', + '--splice-ramp-seconds', '11', + '--shard-count', '4', + '--shard-index', '2' + ]) + + assert.equal(config.spliceRampMs, 11_000) + assert.deepEqual( + [0, 1, 2].map((index) => relayLoadSpliceStartDelayMs(config, index)), + [2_000, 6_000, 10_000] + ) + assert.throws(() => relayLoadSpliceStartDelayMs(config, 3), /invalid local splice index/) +}) + +test('distributes each shard invite wave below the account rate limit', () => { + const identities = Array.from( + { length: 710 }, + (_, localIndex) => relayLoadPrincipalIndex(localIndex * 4 + 2, 4, 32) + ) + const offers = Array.from({ length: 790 }, (_, index) => identities[index % identities.length]) + const counts = new Map() + for (const principal of offers) counts.set(principal, (counts.get(principal) ?? 0) + 1) + assert.equal(counts.size, 32) + assert.equal(Math.max(...counts.values()), 26) +}) + +test('requires exactly one assignment mode', () => { + assert.throws( + () => + parseRelayLoadArguments([ + ...required, + '--target-origin', + 'https://cell.test' + ]), + /exactly one target or director origin/ + ) +}) + +test('requires separate recoverable and wedged reader profiles', () => { + const config = parseRelayLoadArguments([ + ...required, + '--controls', '4', + '--splices', '3', + '--slow-reader-splices', '1', + '--wedged-reader-splices', '1', + '--slow-reader-stream-bytes', '524288', + '--wedged-reader-stream-bytes', '1048576', + '--slow-reader-hold-ms', '9000', + '--wedged-reader-hold-ms', '11000' + ]) + + assert.deepEqual(relayLoadSpliceProfile(config, 0), { + readerMode: 'wedged', readerHoldMs: 11_000, streamBytes: 1_048_576, frameBytes: 65_536 + }) + assert.deepEqual(relayLoadSpliceProfile(config, 1), { + readerMode: 'slow', readerHoldMs: 9_000, streamBytes: 524_288, frameBytes: 65_536 + }) + assert.equal(relayLoadSpliceProfile(config, 2).readerMode, 'normal') +}) + +test('bounds reader stream, timeout, and splice load per shard', () => { + assert.throws( + () => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '3']), + /splices cannot exceed/ + ) + assert.throws( + () => + parseRelayLoadArguments([ + ...required, + '--splices', + '1', + '--slow-reader-splices', + '2' + ]), + /reader splice counts cannot exceed/ + ) + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--splices', '1', '--slow-reader-splices', '1', + '--slow-reader-stream-bytes', '262144' + ]), + /must exceed the 256 KiB/ + ) + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--splices', '1', '--wedged-reader-splices', '1', + '--wedged-reader-stream-bytes', '262144' + ]), + /must exceed the 256 KiB/ + ) + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--splices', '1', '--slow-reader-splices', '1', + '--slow-reader-hold-ms', '10000' + ]), + /must stay below the wedged timeout/ + ) + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--splices', '1', '--wedged-reader-splices', '1', + '--wedged-reader-hold-ms', '10000' + ]), + /must exceed the wedged timeout/ + ) + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--controls', '17', '--splices', '17', '--slow-reader-splices', '17' + ]), + /reader splice count exceeds/ + ) + assert.throws( + () => parseRelayLoadArguments([ + ...required, '--controls', '9', '--splices', '9', '--wedged-reader-splices', '9' + ]), + /reader stream bytes exceed/ + ) +}) + +test('supports one bounded reader-owning shard without multiplying its pressure', () => { + const owner = parseRelayLoadArguments([ + ...required, + '--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '0', + '--aggregate-controls', '2600', '--aggregate-splices', '120', + '--slow-reader-splices', '10', '--wedged-reader-splices', '1', + '--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368' + ]) + const peer = parseRelayLoadArguments([ + ...required, + '--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000', + '--shard-count', '4', '--shard-index', '1', + '--aggregate-controls', '2600', '--aggregate-splices', '120', + '--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368' + ]) + + assert.equal(owner.aggregateReaderSplices, 11) + assert.equal(owner.aggregateReaderBytes, 18 * 1024 * 1024) + assert.equal(peer.slowReaderSplices + peer.wedgedReaderSplices, 0) + assert.equal(peer.aggregateReaderSplices, 11) +}) + +test('requires queue, memory, and expected close evidence', () => { + const config = parseRelayLoadArguments([ + ...required, '--splices', '2', '--slow-reader-splices', '1', + '--wedged-reader-splices', '1', '--max-generator-rss-growth-mib', '100' + ]) + const passing = { + generatorRssGrowthMiB: 50, + slowReaderSplicesCompleted: 1, + wedgedReaderSplicesClosed: 1, + readerQueueEvidence: [ + { origin: 'https://cell.test', baselineBytes: 8, peakBytes: 65_544, increaseBytes: 65_536 } + ], + readerClosesByCode: { '4429': 1 } + } + + assert.equal(relayLoadReaderEvidenceError(passing, config), undefined) + assert.match( + relayLoadReaderEvidenceError({ + ...passing, + readerQueueEvidence: [ + { origin: 'https://cell.test', baselineBytes: 8, peakBytes: 8, increaseBytes: 0 } + ] + }, config), + /no causal Relay queued-byte evidence/ + ) + assert.match( + relayLoadReaderEvidenceError({ ...passing, generatorRssGrowthMiB: 101 }, config), + /RSS growth budget/ + ) + assert.match( + relayLoadReaderEvidenceError({ ...passing, readerClosesByCode: { '4429': 0 } }, config), + /did not close with 4429/ + ) +}) diff --git a/cloud/dev/scripts/relay-load-reader-evidence.mjs b/cloud/dev/scripts/relay-load-reader-evidence.mjs new file mode 100644 index 00000000000..f20b5029587 --- /dev/null +++ b/cloud/dev/scripts/relay-load-reader-evidence.mjs @@ -0,0 +1,51 @@ +const DEFAULT_TIMEOUT_MS = 8_000 +const DEFAULT_POLL_MS = 100 + +export async function createRelayLoadReaderEvidence(origins, dependencies) { + const distinctOrigins = [...new Set(origins)].sort() + const baselines = new Map(await Promise.all(distinctOrigins.map(async (origin) => [ + origin, + await dependencies.readQueuedBytes(origin) + ]))) + const peaks = new Map(baselines) + const pending = new Map() + const now = dependencies.now ?? Date.now + const delay = dependencies.delay + const timeoutMs = dependencies.timeoutMs ?? DEFAULT_TIMEOUT_MS + const pollMs = dependencies.pollMs ?? DEFAULT_POLL_MS + + const observe = async ({ cellOrigin }) => { + if (!baselines.has(cellOrigin)) throw new Error('reader origin lacks a run baseline') + if (peaks.get(cellOrigin) > baselines.get(cellOrigin)) return + const current = pending.get(cellOrigin) + if (current) return await current + const proof = (async () => { + const baseline = baselines.get(cellOrigin) + const deadline = now() + timeoutMs + for (;;) { + const queuedBytes = await dependencies.readQueuedBytes(cellOrigin) + peaks.set(cellOrigin, Math.max(peaks.get(cellOrigin), queuedBytes)) + if (queuedBytes > baseline) return + if (now() >= deadline) { + throw new Error('reader stream produced no causal Relay queued-byte increase') + } + await delay(pollMs) + } + })() + pending.set(cellOrigin, proof) + try { + await proof + } finally { + pending.delete(cellOrigin) + } + } + + const snapshot = () => distinctOrigins.map((origin) => ({ + origin, + baselineBytes: baselines.get(origin), + peakBytes: peaks.get(origin), + increaseBytes: peaks.get(origin) - baselines.get(origin) + })) + + return { observe, snapshot } +} diff --git a/cloud/dev/scripts/relay-load-reader-evidence.test.mjs b/cloud/dev/scripts/relay-load-reader-evidence.test.mjs new file mode 100644 index 00000000000..ce417f5bc42 --- /dev/null +++ b/cloud/dev/scripts/relay-load-reader-evidence.test.mjs @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs' + +test('requires a queue increase above the pre-injection baseline for every origin', async () => { + const samples = new Map([ + ['https://a.test', [7, 7, 11]], + ['https://b.test', [0, 3]] + ]) + let now = 0 + const evidence = await createRelayLoadReaderEvidence([...samples.keys()], { + readQueuedBytes: async (origin) => samples.get(origin).shift(), + delay: async (ms) => { now += ms }, + now: () => now + }) + + await Promise.all([ + evidence.observe({ cellOrigin: 'https://a.test' }), + evidence.observe({ cellOrigin: 'https://b.test' }) + ]) + + assert.deepEqual(evidence.snapshot(), [ + { origin: 'https://a.test', baselineBytes: 7, peakBytes: 11, increaseBytes: 4 }, + { origin: 'https://b.test', baselineBytes: 0, peakBytes: 3, increaseBytes: 3 } + ]) +}) + +test('shares one causal proof across concurrent readers on the same cell', async () => { + const samples = [4, 4, 9] + let reads = 0 + let now = 0 + const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], { + readQueuedBytes: async () => { reads++; return samples.shift() }, + delay: async (ms) => { now += ms }, + now: () => now + }) + + await Promise.all([ + evidence.observe({ cellOrigin: 'https://cell.test' }), + evidence.observe({ cellOrigin: 'https://cell.test' }) + ]) + + assert.equal(reads, 3) + assert.equal(evidence.snapshot()[0].increaseBytes, 5) +}) + +test('reuses a completed causal proof for later readers on the same cell', async () => { + const samples = [4, 9] + let reads = 0 + const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], { + readQueuedBytes: async () => { reads++; return samples.shift() }, + delay: async () => undefined + }) + + await evidence.observe({ cellOrigin: 'https://cell.test' }) + await evidence.observe({ cellOrigin: 'https://cell.test' }) + + assert.equal(reads, 2) + assert.equal(evidence.snapshot()[0].increaseBytes, 5) +}) + +test('rejects a pre-existing nonzero queue that never increases', async () => { + let now = 0 + const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], { + readQueuedBytes: async () => 9, + delay: async (ms) => { now += ms }, + now: () => now, + timeoutMs: 200, + pollMs: 100 + }) + + await assert.rejects( + evidence.observe({ cellOrigin: 'https://cell.test' }), + /no causal Relay queued-byte increase/ + ) +}) diff --git a/cloud/dev/scripts/relay-load-rebind-boundary.mjs b/cloud/dev/scripts/relay-load-rebind-boundary.mjs new file mode 100644 index 00000000000..2eb72c7d6ba --- /dev/null +++ b/cloud/dev/scripts/relay-load-rebind-boundary.mjs @@ -0,0 +1,59 @@ +export async function waitForRelayLoadRebindGate({ + delay, + delayMs, + activeCount, + requiredCount +}) { + await delay(delayMs) + const active = activeCount() + if (active !== requiredCount) { + throw new Error(`rebind boundary requires ${requiredCount} active controls, found ${active}`) + } +} + +export async function proveRelayLoadRebindBoundary({ + peers, + probeCount, + holdMs, + delay, + failureReason, + requireOverflow = true +}) { + if (probeCount === 0) return { opened: 0, overflowReason: null } + if (peers.length < probeCount) throw new Error('insufficient active controls for rebind proof') + + const probes = [] + try { + const opened = await Promise.allSettled( + peers.slice(0, probeCount).map((peer) => peer.openRebindProbe()) + ) + for (const result of opened) { + if (result.status === 'fulfilled') probes.push(result.value) + } + const rejected = opened.find((result) => result.status === 'rejected') + if (rejected) throw rejected.reason + + let overflowReason = null + if (requireOverflow) { + try { + const overflow = await peers[0].openRebindProbe() + await overflow.close() + } catch (error) { + overflowReason = failureReason(error) + } + if (overflowReason !== 'socket_http_503') { + throw new Error(`rebind overflow was not rejected at the hard cap: ${overflowReason}`) + } + } + const closedIndex = await Promise.race([ + delay(holdMs).then(() => -1), + ...probes.map((probe, index) => probe.closed.then(() => index)) + ]) + if (closedIndex >= 0 || probes.some((probe) => !probe.isOpen())) { + throw new Error('rebind probe closed before the hold completed') + } + return { opened: probes.length, overflowReason } + } finally { + await Promise.all(probes.map((probe) => probe.close())) + } +} diff --git a/cloud/dev/scripts/relay-load-rebind-boundary.test.mjs b/cloud/dev/scripts/relay-load-rebind-boundary.test.mjs new file mode 100644 index 00000000000..50ad2a48383 --- /dev/null +++ b/cloud/dev/scripts/relay-load-rebind-boundary.test.mjs @@ -0,0 +1,164 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + proveRelayLoadRebindBoundary, + waitForRelayLoadRebindGate +} from './relay-load-rebind-boundary.mjs' + +function peer(open) { + return { openRebindProbe: open } +} + +function probe(onClose = () => undefined) { + let open = true + let resolveClosed + const closed = new Promise((resolve) => { + resolveClosed = resolve + }) + return { + close: () => { + if (!open) return + open = false + onClose() + resolveClosed() + }, + closed, + isOpen: () => open + } +} + +test('holds the requested rebind overlap and requires a hard-cap rejection', async () => { + let openCalls = 0 + let closes = 0 + let heldFor = null + const peers = [ + peer(async () => { + openCalls++ + if (openCalls === 3) throw new Error('Unexpected server response: 503') + return probe(() => closes++) + }), + peer(async () => { + openCalls++ + return probe(() => closes++) + }) + ] + const result = await proveRelayLoadRebindBoundary({ + peers, + probeCount: 2, + holdMs: 4_000, + delay: async (milliseconds) => { + heldFor = milliseconds + }, + failureReason: (error) => + error.message.includes('503') ? 'socket_http_503' : 'unknown' + }) + + assert.deepEqual(result, { opened: 2, overflowReason: 'socket_http_503' }) + assert.equal(heldFor, 4_000) + assert.equal(closes, 2) +}) + +test('closes successful probes when a boundary probe fails', async () => { + let closes = 0 + const peers = [ + peer(async () => probe(() => closes++)), + peer(async () => { + throw new Error('probe failed') + }) + ] + + await assert.rejects( + proveRelayLoadRebindBoundary({ + peers, + probeCount: 2, + holdMs: 0, + delay: async () => undefined, + failureReason: () => 'unknown' + }), + /probe failed/ + ) + assert.equal(closes, 1) +}) + +test('waits for every replacement socket to finish closing', async () => { + let finishClose + const closeFinished = new Promise((resolve) => { + finishClose = resolve + }) + const closingProbe = probe() + closingProbe.close = () => closeFinished + let completed = false + const boundary = proveRelayLoadRebindBoundary({ + peers: [peer(async () => closingProbe)], + probeCount: 1, + holdMs: 0, + delay: async () => undefined, + failureReason: () => 'unknown', + requireOverflow: false + }).then(() => { + completed = true + }) + + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(completed, false) + finishClose() + await boundary + assert.equal(completed, true) +}) + +test('can prove reserved replacement headroom below the physical cap', async () => { + let closes = 0 + const result = await proveRelayLoadRebindBoundary({ + peers: [peer(async () => probe(() => closes++))], + probeCount: 1, + holdMs: 0, + delay: async () => undefined, + failureReason: () => 'unknown', + requireOverflow: false + }) + assert.deepEqual(result, { opened: 1, overflowReason: null }) + assert.equal(closes, 1) +}) + +test('fails when a replacement closes before the hold completes', async () => { + let heldProbe + await assert.rejects( + proveRelayLoadRebindBoundary({ + peers: [ + peer(async () => { + heldProbe = probe() + return heldProbe + }) + ], + probeCount: 1, + holdMs: 4_000, + delay: async () => { + heldProbe.close() + }, + failureReason: () => 'unknown', + requireOverflow: false + }), + /closed before the hold completed/ + ) +}) + +test('delays the boundary until every ordinary control has recovered', async () => { + let active = 899 + await assert.rejects( + waitForRelayLoadRebindGate({ + delay: async () => undefined, + delayMs: 0, + activeCount: () => active, + requiredCount: 900 + }), + /requires 900 active controls/ + ) + await waitForRelayLoadRebindGate({ + delay: async () => { + active = 900 + }, + delayMs: 1, + activeCount: () => active, + requiredCount: 900 + }) +}) diff --git a/cloud/dev/scripts/relay-load-region-behavior.mjs b/cloud/dev/scripts/relay-load-region-behavior.mjs new file mode 100644 index 00000000000..b0333b32abe --- /dev/null +++ b/cloud/dev/scripts/relay-load-region-behavior.mjs @@ -0,0 +1,35 @@ +const ASSIGNMENT_RETRY_DELAY_MS = 5_100 + +const waitPastAssignmentRateLimit = (schedule) => + new Promise((resolve) => schedule(resolve, ASSIGNMENT_RETRY_DELAY_MS)) + +export async function proveRelayLoadRegionBehavior({ + oldClientPeer, + stickyPeer, + asiaOrigin, + scheduleAssignmentRetry = setTimeout +}) { + try { + await oldClientPeer.connect() + if (!oldClientPeer.assignedCellUrl() || oldClientPeer.assignedCellUrl() === asiaOrigin) { + throw new Error('unhinted client did not use the US-first path') + } + } finally { + await oldClientPeer.shutdown() + } + + try { + await stickyPeer.connect() + if (stickyPeer.assignedCellUrl() !== asiaOrigin) { + throw new Error('preferred Asia client did not reach the Asia cell') + } + await waitPastAssignmentRateLimit(scheduleAssignmentRetry) + const reassigned = await stickyPeer.requestAssignment('us-central1') + if (reassigned.cellUrl !== asiaOrigin) { + throw new Error('valid sticky assignment moved after preference changed') + } + } finally { + await stickyPeer.shutdown() + } + return { oldClientUsFirst: true, stickyAssignmentPreserved: true } +} diff --git a/cloud/dev/scripts/relay-load-region-behavior.test.mjs b/cloud/dev/scripts/relay-load-region-behavior.test.mjs new file mode 100644 index 00000000000..5882e3021a1 --- /dev/null +++ b/cloud/dev/scripts/relay-load-region-behavior.test.mjs @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs' + +function peer(origin, reassigned = origin) { + let shutdowns = 0 + return { + connect: async () => undefined, + assignedCellUrl: () => origin, + requestAssignment: async () => ({ cellUrl: reassigned }), + shutdown: async () => { shutdowns++ }, + shutdowns: () => shutdowns + } +} + +test('proves unhinted US-first placement and sticky Asia preservation', async () => { + const oldClientPeer = peer('https://c3.relay-staging.onorca.dev') + const stickyPeer = peer('https://c4.relay-staging.onorca.dev') + let retryDelayMs = 0 + assert.deepEqual(await proveRelayLoadRegionBehavior({ + oldClientPeer, + stickyPeer, + asiaOrigin: 'https://c4.relay-staging.onorca.dev', + scheduleAssignmentRetry: (resolve, delayMs) => { + retryDelayMs = delayMs + resolve() + } + }), { oldClientUsFirst: true, stickyAssignmentPreserved: true }) + assert.equal(retryDelayMs, 5_100) + assert.equal(oldClientPeer.shutdowns(), 1) + assert.equal(stickyPeer.shutdowns(), 1) +}) + +test('rejects Asia placement for an unhinted client or a moved sticky assignment', async () => { + await assert.rejects(proveRelayLoadRegionBehavior({ + oldClientPeer: peer('https://c4.relay-staging.onorca.dev'), + stickyPeer: peer('https://c4.relay-staging.onorca.dev'), + asiaOrigin: 'https://c4.relay-staging.onorca.dev', + scheduleAssignmentRetry: (resolve) => resolve() + }), /US-first/) + await assert.rejects(proveRelayLoadRegionBehavior({ + oldClientPeer: peer('https://c3.relay-staging.onorca.dev'), + stickyPeer: peer('https://c4.relay-staging.onorca.dev', 'https://c3.relay-staging.onorca.dev'), + asiaOrigin: 'https://c4.relay-staging.onorca.dev', + scheduleAssignmentRetry: (resolve) => resolve() + }), /sticky assignment moved/) +}) diff --git a/cloud/dev/scripts/relay-load-request-unit-boundary.mjs b/cloud/dev/scripts/relay-load-request-unit-boundary.mjs new file mode 100644 index 00000000000..a5dbb59d96b --- /dev/null +++ b/cloud/dev/scripts/relay-load-request-unit-boundary.mjs @@ -0,0 +1,43 @@ +import { setTimeout as delayDefault } from 'node:timers/promises' + +export async function openRelayLoadInviteOffers({ + peers, + count, + ratePerSecond, + concurrency = 8, + delay = delayDefault, + now = Date.now +}) { + if ( + !Array.isArray(peers) || peers.length === 0 || + !Number.isSafeInteger(count) || count < 0 || + !Number.isSafeInteger(ratePerSecond) || ratePerSecond < 1 || ratePerSecond > 20 || + !Number.isSafeInteger(concurrency) || concurrency < 1 + ) throw new Error('invalid Relay invite-offer load') + let next = 0 + let nextStartAt = now() + const workers = Array.from({ length: Math.min(concurrency, count) }, async () => { + for (;;) { + const index = next++ + if (index >= count) return + const scheduledAt = Math.max(nextStartAt, now()) + nextStartAt = scheduledAt + 1_000 / ratePerSecond + await delay(Math.max(0, scheduledAt - now())) + await peers[index % peers.length].openInviteOffer() + } + }) + await Promise.all(workers) + return count +} + +export async function proveRelayLoadRequestUnitBoundary(peer) { + try { + await peer.openInviteOffer() + } catch (error) { + if (error instanceof Error && error.message === 'invite offer failed: relay_capacity_exhausted') { + return 'relay_capacity_exhausted' + } + throw new Error('request-unit overflow was not rejected safely', { cause: error }) + } + throw new Error('request-unit overflow unexpectedly succeeded') +} diff --git a/cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs b/cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs new file mode 100644 index 00000000000..739e055c93a --- /dev/null +++ b/cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + openRelayLoadInviteOffers, + proveRelayLoadRequestUnitBoundary +} from './relay-load-request-unit-boundary.mjs' + +test('distributes the exact invite count with bounded concurrency', async () => { + let active = 0 + let peak = 0 + const delays = [] + const calls = [0, 0, 0] + const peers = calls.map((_, index) => ({ + async openInviteOffer() { + calls[index]++ + active++ + peak = Math.max(peak, active) + await Promise.resolve() + active-- + } + })) + assert.equal(await openRelayLoadInviteOffers({ + peers, + count: 8, + ratePerSecond: 2, + concurrency: 2, + delay: async (milliseconds) => { delays.push(milliseconds) }, + now: () => 0 + }), 8) + assert.deepEqual(calls, [3, 3, 2]) + assert.ok(peak <= 2) + assert.equal(delays.length, 8) + assert.ok(Math.max(...delays) >= 3_500) +}) + +test('accepts only the exact request-unit exhaustion error', async () => { + assert.equal(await proveRelayLoadRequestUnitBoundary({ + openInviteOffer: async () => { + throw new Error('invite offer failed: relay_capacity_exhausted') + } + }), 'relay_capacity_exhausted') + await assert.rejects( + proveRelayLoadRequestUnitBoundary({ + openInviteOffer: async () => { throw new Error('control response timeout') } + }), + /not rejected safely/ + ) + await assert.rejects( + proveRelayLoadRequestUnitBoundary({ openInviteOffer: async () => undefined }), + /unexpectedly succeeded/ + ) +}) diff --git a/cloud/dev/scripts/relay-load-run-lifecycle.mjs b/cloud/dev/scripts/relay-load-run-lifecycle.mjs new file mode 100644 index 00000000000..dffd76a813b --- /dev/null +++ b/cloud/dev/scripts/relay-load-run-lifecycle.mjs @@ -0,0 +1,25 @@ +export function assertRelayLoadRampAccepted(rampConnectionFailures, maximum) { + if (rampConnectionFailures > maximum) { + throw new Error('relay load ramp exceeded the allowed connection failures') + } +} + +export async function runRelayLoadWithShutdown(operation, shutdown) { + try { + return await operation() + } finally { + await shutdown() + } +} + +export function relayLoadRunHasDisallowedFailures(result, config) { + return ( + result.rampConnectionFailures > config.maxRampConnectionFailures || + (!config.allowPlannedTransitionRetries && result.transitionConnectionFailures > 0) || + result.steadyConnectionFailures > 0 || + result.unexpectedCloses > config.maxUnexpectedCloses || + result.protocolErrors > 0 || + result.refreshErrors > 0 || + result.socketErrors > 0 + ) +} diff --git a/cloud/dev/scripts/relay-load-run-lifecycle.test.mjs b/cloud/dev/scripts/relay-load-run-lifecycle.test.mjs new file mode 100644 index 00000000000..5aec8e08ab9 --- /dev/null +++ b/cloud/dev/scripts/relay-load-run-lifecycle.test.mjs @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + assertRelayLoadRampAccepted, + relayLoadRunHasDisallowedFailures, + runRelayLoadWithShutdown +} from './relay-load-run-lifecycle.mjs' + +test('always shuts down peers when a load phase fails', async () => { + const events = [] + await assert.rejects( + runRelayLoadWithShutdown( + async () => { + events.push('run') + throw new Error('boundary failed') + }, + async () => events.push('shutdown') + ), + /boundary failed/ + ) + assert.deepEqual(events, ['run', 'shutdown']) +}) + +test('fails immediately when the strict ramp budget is exceeded', () => { + assert.doesNotThrow(() => assertRelayLoadRampAccepted(0, 0)) + assert.throws(() => assertRelayLoadRampAccepted(1, 0), /ramp exceeded/) +}) + +test('rejects connection failures during the transition window', () => { + const result = { + rampConnectionFailures: 0, + transitionConnectionFailures: 1, + steadyConnectionFailures: 0, + unexpectedCloses: 0, + protocolErrors: 0, + refreshErrors: 0, + socketErrors: 0 + } + assert.equal( + relayLoadRunHasDisallowedFailures(result, { + maxRampConnectionFailures: 0, + maxUnexpectedCloses: 0 + }), + true + ) +}) + +test('allows only explicitly planned transition retries', () => { + const result = { + rampConnectionFailures: 0, + transitionConnectionFailures: 1, + steadyConnectionFailures: 0, + unexpectedCloses: 0, + protocolErrors: 0, + refreshErrors: 0, + socketErrors: 0 + } + const config = { + allowPlannedTransitionRetries: true, + maxRampConnectionFailures: 0, + maxUnexpectedCloses: 0 + } + assert.equal(relayLoadRunHasDisallowedFailures(result, config), false) + assert.equal( + relayLoadRunHasDisallowedFailures({ ...result, steadyConnectionFailures: 1 }, config), + true + ) +}) diff --git a/cloud/dev/scripts/relay-monitor-evidence.mjs b/cloud/dev/scripts/relay-monitor-evidence.mjs new file mode 100644 index 00000000000..7f387663f60 --- /dev/null +++ b/cloud/dev/scripts/relay-monitor-evidence.mjs @@ -0,0 +1,355 @@ +import { createHash } from 'node:crypto' +import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises' +import { basename, join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/ +const SHA = /^[a-f0-9]{40}$/ +const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/ +const EVIDENCE_MAX_AGE_MS = 5 * 60_000 +// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave. +const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000 +const WAVE_INDEX = /^[0-3]$/ +const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000 +const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000 +const MIGRATION_POLICIES = new Set([ + 'strict', + 'recover-forward', + 'capacity-transition' +]) +const MUTATION_MODES = new Set([ + 'capacity-transition', + 'continue-evacuation', + 'disable-cell', + 'enable-empty-cell', + 'execute', + 'fence-source', + 'recover-forward', + 'reset-empty-candidate' +]) + +function argumentsByName(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const name = argv[index] + const value = argv[index + 1] + if (!name?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error('relay monitor evidence arguments are invalid') + } + values[name.slice(2)] = value + } + return values +} + +async function sha256(path) { + return createHash('sha256').update(await readFile(path)).digest('hex') +} + +async function regularFile(path) { + try { + return (await stat(path)).isFile() + } catch { + return false + } +} + +function provenance(values) { + const runAttempt = Number(values['run-attempt']) + if ( + !SAFE_ID.test(values['incident-id'] ?? '') || + !SAFE_ID.test(values['run-id'] ?? '') || + !Number.isSafeInteger(runAttempt) || + runAttempt < 1 || + !SHA.test(values['commit-sha'] ?? '') || + !['dry-run', 'monitor'].includes(values.mode) + ) { + throw new Error('relay monitor evidence provenance is invalid') + } + return { + incidentId: values['incident-id'], + runId: values['run-id'], + runAttempt, + commitSha: values['commit-sha'], + mode: values.mode + } +} + +export async function createEvidenceManifest(argv) { + const values = argumentsByName(argv) + const directory = resolve(values.directory ?? '') + const expected = provenance(values) + const candidates = [ + `${expected.incidentId}.state.json`, + `${expected.incidentId}.summaries.jsonl`, + `${expected.incidentId}.summary.md` + ] + const files = {} + for (const name of candidates) { + const path = join(directory, name) + if (await regularFile(path)) files[name] = await sha256(path) + } + if (!files[`${expected.incidentId}.state.json`]) { + throw new Error('relay monitor durable state is missing') + } + const manifest = { + schemaVersion: 1, + ...expected, + files + } + const path = join(directory, 'evidence-manifest.json') + await writeFile(path, `${JSON.stringify(manifest)}\n`, { mode: 0o600 }) + await chmod(path, 0o600) + return manifest +} + +async function readAndVerifyManifest(directory, expected) { + const manifest = JSON.parse( + await readFile(join(directory, 'evidence-manifest.json'), 'utf8') + ) + if ( + manifest.schemaVersion !== 1 || + manifest.incidentId !== expected.incidentId || + manifest.runId !== expected.runId || + manifest.runAttempt !== expected.runAttempt || + manifest.commitSha !== expected.commitSha || + manifest.mode !== expected.mode + ) { + throw new Error('relay monitor evidence provenance does not match') + } + const names = Object.keys(manifest.files ?? {}) + if (!names.includes(`${expected.incidentId}.state.json`)) { + throw new Error('relay monitor evidence has no durable state') + } + for (const name of names) { + if (basename(name) !== name || !/^[A-Za-z0-9._-]+$/.test(name)) { + throw new Error('relay monitor evidence file name is invalid') + } + if (await sha256(join(directory, name)) !== manifest.files[name]) { + throw new Error('relay monitor evidence hash does not match') + } + } + const allowed = new Set([...names, 'evidence-manifest.json']) + const unexpected = (await readdir(directory)).filter((name) => !allowed.has(name)) + if (unexpected.length > 0) throw new Error('relay monitor evidence has unexpected files') + return manifest +} + +function validMigrationPolicyState(state) { + return ( + ( + state.migrationPolicy === 'strict' && + state.recoverySourceCellId === null && + state.capacityCellId === null + ) || + ( + state.migrationPolicy === 'recover-forward' && + state.capacityCellId === null && + typeof state.recoverySourceCellId === 'string' && + state.expectedSelector?.membership?.existingOnly?.includes( + state.recoverySourceCellId + ) + ) || + ( + state.migrationPolicy === 'capacity-transition' && + state.recoverySourceCellId === null && + typeof state.capacityCellId === 'string' && + state.expectedSelector?.membership?.general?.includes(state.capacityCellId) + ) + ) +} + +export async function verifyRestoredEvidence(argv) { + const values = argumentsByName(argv) + const directory = resolve(values.directory ?? '') + const expected = provenance(values) + await readAndVerifyManifest(directory, expected) + const state = JSON.parse( + await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8') + ) + if ( + state.schemaVersion !== 4 || + state.incidentId !== expected.incidentId || + state.environment !== 'production' || + state.preDrainDryRun !== (expected.mode === 'dry-run') || + !MIGRATION_POLICIES.has(state.migrationPolicy) || + !validMigrationPolicyState(state) + ) { + throw new Error('relay monitor restored state does not match provenance') + } + return state +} + +function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) { + const completedAt = Date.parse(state.completedAt) + const startedAt = Date.parse(state.startedAt) + const windowStartedAt = Date.parse(state.windowStartedAt) + const lastSampleAt = Date.parse(state.lastSampleAt) + const age = nowMs - completedAt + return ( + state.schemaVersion === 4 && + state.incidentId === expected.incidentId && + state.environment === 'production' && + state.preDrainDryRun === true && + validMigrationPolicyState(state) && + state.durationMinutes === 15 && + state.intervalMs === EVIDENCE_SAMPLE_INTERVAL_MS && + state.sampleCount >= 16 && + state.frozenAt === null && + Number.isFinite(startedAt) && + completedAt - startedAt >= 0 && + completedAt - startedAt <= EVIDENCE_MAX_LINEAGE_MS && + Number.isFinite(windowStartedAt) && + completedAt - windowStartedAt >= 15 * 60_000 && + Number.isFinite(lastSampleAt) && + lastSampleAt <= completedAt && + completedAt - lastSampleAt <= state.intervalMs && + Number.isFinite(completedAt) && + age >= 0 && + age <= maxAgeMs + ) +} + +export async function verifyDryRunAuthority(argv, now = Date.now) { + const values = argumentsByName(argv) + const directory = resolve(values.directory ?? '') + const expected = provenance(values) + if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence') + const manifest = await readAndVerifyManifest(directory, expected) + const state = JSON.parse( + await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8') + ) + const requiredMigrationPolicy = values['required-migration-policy'] + // Later same-cap waves start after sequential predecessor cell rolls, so the + // freshness bound grows by one cell-job timeout per predecessor; single-use + // consumption, needs-chaining, and each wave's live preflight recheck keep + // holding the mutation to current health. + const waveIndex = values['wave-index'] ?? '0' + if (!WAVE_INDEX.test(waveIndex)) { + throw new Error('relay monitor wave index is invalid') + } + const maxAgeMs = + EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS + if ( + !MIGRATION_POLICIES.has(requiredMigrationPolicy) || + state.migrationPolicy !== requiredMigrationPolicy || + !validCompletedDryRunState(state, expected, now(), maxAgeMs) + ) { + throw new Error('relay monitor dry-run authority is incomplete or stale') + } + return { manifest, state } +} + +function exactSelector(actual, expected) { + const membership = (selector) => { + if ( + !selector?.membership || + !['existingOnly', 'migrationOnly', 'general'].every((key) => + Array.isArray(selector.membership[key]) + ) + ) return null + const normalized = Object.fromEntries( + ['existingOnly', 'migrationOnly', 'general'].map((key) => [ + key, + [...selector.membership[key]].sort() + ]) + ) + const all = Object.values(normalized).flat() + return new Set(all).size === all.length ? normalized : null + } + const actualMembership = membership(actual) + const expectedMembership = membership(expected) + return Boolean( + actualMembership && + expectedMembership && + actual?.generation === expected?.generation && + JSON.stringify(actualMembership) === JSON.stringify(expectedMembership) + ) +} + +export async function verifyMutationEvidence( + argv, + environment = process.env, + fetchImpl = fetch, + now = Date.now +) { + const values = argumentsByName(argv) + const directory = resolve(values.directory ?? '') + const expected = provenance(values) + if (expected.mode !== 'dry-run') throw new Error('mutation requires dry-run evidence') + const manifest = await readAndVerifyManifest(directory, expected) + const state = JSON.parse( + await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8') + ) + const mutationMode = values['mutation-mode'] + if (!MUTATION_MODES.has(mutationMode)) { + throw new Error('relay monitor mutation mode is invalid') + } + const scopedRecoverySourceCellId = + values['scoped-recovery-source-cell-id'] + const recoveryMutation = ['fence-source', 'recover-forward'].includes(mutationMode) + const scopedRecoveryMutation = + ['execute', 'recover-forward'].includes(mutationMode) && + Boolean(scopedRecoverySourceCellId) + if (scopedRecoverySourceCellId && !scopedRecoveryMutation) { + throw new Error('relay monitor scoped recovery evidence is invalid') + } + const requiredMigrationPolicy = mutationMode === 'capacity-transition' + ? 'capacity-transition' + : recoveryMutation || scopedRecoveryMutation ? 'recover-forward' : 'strict' + if (state.migrationPolicy !== requiredMigrationPolicy) { + throw new Error('relay monitor migration policy does not match mutation') + } + const expectedRecoverySourceCellId = scopedRecoveryMutation + ? scopedRecoverySourceCellId + : values['source-cell-id'] + if ( + (recoveryMutation || scopedRecoveryMutation) && + state.recoverySourceCellId !== expectedRecoverySourceCellId + ) { + throw new Error('relay monitor recovery source does not match mutation') + } + if ( + mutationMode === 'capacity-transition' && + state.capacityCellId !== values['source-cell-id'] + ) { + throw new Error('relay monitor capacity cell does not match mutation') + } + if ( + !validCompletedDryRunState(state, expected, now(), EVIDENCE_MAX_AGE_MS) + ) { + throw new Error('relay monitor dry-run evidence is incomplete or stale') + } + const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN + const origin = values['director-origin'] + if (!token || !JWT.test(token) || !origin?.startsWith('https://')) { + throw new Error('relay monitor live selector verification is unavailable') + } + const response = await fetchImpl(`${origin}/v1/admin/admission-selector/status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1 }), + signal: AbortSignal.timeout(30_000) + }) + if (!response.ok) throw new Error('relay monitor live selector verification failed') + const current = (await response.json()).selector + if (!exactSelector(current, state.expectedSelector)) { + throw new Error('relay admission selector changed after the dry run') + } + return { manifest, state } +} + +async function main() { + const [command, ...argv] = process.argv.slice(2) + if (command === 'create') await createEvidenceManifest(argv) + else if (command === 'verify-restore') await verifyRestoredEvidence(argv) + else if (command === 'verify-authority') await verifyDryRunAuthority(argv) + else if (command === 'verify-mutation') await verifyMutationEvidence(argv) + else throw new Error('relay monitor evidence command is invalid') +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : 'relay monitor evidence failed') + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/relay-monitor-evidence.test.mjs b/cloud/dev/scripts/relay-monitor-evidence.test.mjs new file mode 100644 index 00000000000..45116761119 --- /dev/null +++ b/cloud/dev/scripts/relay-monitor-evidence.test.mjs @@ -0,0 +1,515 @@ +import assert from 'node:assert/strict' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { relayWorkflowPath, relayWorkflowUrl } from './relay-repository.mjs' +import { + createEvidenceManifest, + verifyDryRunAuthority, + verifyMutationEvidence, + verifyRestoredEvidence +} from './relay-monitor-evidence.mjs' + +const now = Date.parse('2026-07-28T12:00:00.000Z') +const provenance = [ + '--incident-id', + 'relay-123', + '--run-id', + '123', + '--run-attempt', + '1', + '--commit-sha', + 'a'.repeat(40), + '--mode', + 'dry-run' +] +const selector = { + generation: 2, + membership: { + existingOnly: ['c1'], + migrationOnly: ['c2'], + general: ['c3'] + } +} + +async function evidenceDirectory(migrationPolicy = 'strict') { + const directory = await mkdtemp(join(tmpdir(), 'relay-monitor-evidence-')) + const state = { + schemaVersion: 4, + incidentId: 'relay-123', + environment: 'production', + preDrainDryRun: true, + migrationPolicy, + recoverySourceCellId: migrationPolicy === 'recover-forward' ? 'c1' : null, + capacityCellId: migrationPolicy === 'capacity-transition' ? 'c3' : null, + startedAt: new Date(now - 17 * 60_000).toISOString(), + durationMinutes: 15, + intervalMs: 60_000, + sampleCount: 16, + windowStartedAt: new Date(now - 16 * 60_000).toISOString(), + lastSampleAt: new Date(now - 60_007).toISOString(), + completedAt: new Date(now - 60_000).toISOString(), + frozenAt: null, + expectedSelector: selector + } + await writeFile( + join(directory, 'relay-123.state.json'), + `${JSON.stringify(state)}\n` + ) + return directory +} + +test('creates and verifies exact restart provenance and hashes', async () => { + const directory = await evidenceDirectory() + try { + await createEvidenceManifest(['--directory', directory, ...provenance]) + await assert.doesNotReject( + verifyRestoredEvidence(['--directory', directory, ...provenance]) + ) + await writeFile(join(directory, 'relay-123.state.json'), '{}\n') + await assert.rejects( + verifyRestoredEvidence(['--directory', directory, ...provenance]), + /hash does not match/ + ) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test('requires fresh green evidence and rechecks the live selector', async () => { + const directory = await evidenceDirectory() + try { + await createEvidenceManifest(['--directory', directory, ...provenance]) + const verifyAuthority = () => verifyDryRunAuthority( + [ + '--directory', + directory, + ...provenance, + '--required-migration-policy', + 'strict' + ], + () => now + ) + await assert.doesNotReject(verifyAuthority()) + const fetchImpl = async (_input, init) => { + assert.equal( + new Headers(init.headers).get('authorization'), + 'Bearer aaa.bbb.ccc' + ) + return Response.json({ selector }) + } + await assert.doesNotReject( + verifyMutationEvidence( + [ + '--directory', + directory, + ...provenance, + '--mutation-mode', + 'execute', + '--source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ) + ) + const statePath = join(directory, 'relay-123.state.json') + const state = JSON.parse(await readFile(statePath, 'utf8')) + state.completedAt = new Date(now - 300_001).toISOString() + await writeFile(statePath, `${JSON.stringify(state)}\n`) + await createEvidenceManifest(['--directory', directory, ...provenance]) + await assert.rejects( + verifyAuthority(), + /authority is incomplete or stale/ + ) + await assert.rejects( + verifyMutationEvidence( + [ + '--directory', + directory, + ...provenance, + '--mutation-mode', + 'execute', + '--source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ), + /incomplete or stale/ + ) + state.completedAt = new Date(now - 60_000).toISOString() + state.lastSampleAt = new Date(now - 120_001).toISOString() + await writeFile(statePath, `${JSON.stringify(state)}\n`) + await createEvidenceManifest(['--directory', directory, ...provenance]) + await assert.rejects( + verifyMutationEvidence( + [ + '--directory', + directory, + ...provenance, + '--mutation-mode', + 'execute', + '--source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ), + /incomplete or stale/ + ) + state.lastSampleAt = new Date(now - 60_007).toISOString() + state.startedAt = new Date(now - 26 * 60_000 - 1).toISOString() + await writeFile(statePath, `${JSON.stringify(state)}\n`) + await createEvidenceManifest(['--directory', directory, ...provenance]) + await assert.rejects( + verifyAuthority(), + /authority is incomplete or stale/ + ) + await assert.rejects( + verifyMutationEvidence( + [ + '--directory', + directory, + ...provenance, + '--mutation-mode', + 'execute', + '--source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ), + /incomplete or stale/ + ) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test('later same-cap waves accept evidence aged by predecessor cell rolls', async () => { + const directory = await evidenceDirectory() + try { + const statePath = join(directory, 'relay-123.state.json') + const state = JSON.parse(await readFile(statePath, 'utf8')) + const authorityAt = (...waveArgs) => verifyDryRunAuthority( + [ + '--directory', + directory, + ...provenance, + '--required-migration-policy', + 'strict', + ...waveArgs.flatMap((waveIndex) => ['--wave-index', waveIndex]) + ], + () => now + ) + const ageState = async (ageMs) => { + state.completedAt = new Date(now - ageMs).toISOString() + state.lastSampleAt = new Date(now - ageMs - 7).toISOString() + state.startedAt = new Date(now - ageMs - 17 * 60_000).toISOString() + state.windowStartedAt = new Date(now - ageMs - 16 * 60_000).toISOString() + await writeFile(statePath, `${JSON.stringify(state)}\n`) + await createEvidenceManifest(['--directory', directory, ...provenance]) + } + // The wave-0 bound in isolation: exactly 5 minutes, flag or no flag. + await ageState(5 * 60_000) + await assert.doesNotReject(authorityAt()) + await assert.doesNotReject(authorityAt('0')) + await ageState(5 * 60_000 + 1) + await assert.rejects(authorityAt(), /authority is incomplete or stale/) + await assert.rejects(authorityAt('0'), /authority is incomplete or stale/) + // One predecessor cell roll (~16 min) exceeds wave 0 but fits wave 1. + await ageState(17 * 60_000) + await assert.rejects(authorityAt('0'), /authority is incomplete or stale/) + await assert.doesNotReject(authorityAt('1')) + await assert.rejects(authorityAt('4'), /wave index is invalid/) + await assert.rejects(authorityAt('x'), /wave index is invalid/) + // Both edges of one predecessor job timeout: 5min + 75min exactly. + await ageState(80 * 60_000) + await assert.doesNotReject(authorityAt('1')) + await ageState(80 * 60_000 + 1) + await assert.rejects(authorityAt('1'), /authority is incomplete or stale/) + await assert.doesNotReject(authorityAt('2')) + // Wave 2 and wave 3 edges: 5min + 2 * 75min and 5min + 3 * 75min exactly. + await ageState(155 * 60_000) + await assert.doesNotReject(authorityAt('2')) + await ageState(155 * 60_000 + 1) + await assert.rejects(authorityAt('2'), /authority is incomplete or stale/) + await ageState(230 * 60_000) + await assert.doesNotReject(authorityAt('3')) + await ageState(230 * 60_000 + 1) + await assert.rejects(authorityAt('3'), /authority is incomplete or stale/) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test('binds migration policies to their exact mutations', async () => { + const strictDirectory = await evidenceDirectory() + const recoveryDirectory = await evidenceDirectory('recover-forward') + const capacityDirectory = await evidenceDirectory('capacity-transition') + const fetchImpl = async () => Response.json({ selector }) + try { + await createEvidenceManifest(['--directory', strictDirectory, ...provenance]) + await createEvidenceManifest(['--directory', recoveryDirectory, ...provenance]) + await createEvidenceManifest(['--directory', capacityDirectory, ...provenance]) + await assert.rejects( + verifyDryRunAuthority( + [ + '--directory', + recoveryDirectory, + ...provenance, + '--required-migration-policy', + 'strict' + ], + () => now + ), + /authority is incomplete or stale/ + ) + const verify = (directory, mutationMode, sourceCellId = 'c1') => verifyMutationEvidence( + [ + '--directory', + directory, + ...provenance, + '--mutation-mode', + mutationMode, + '--source-cell-id', + sourceCellId, + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ) + await assert.doesNotReject(verify(strictDirectory, 'execute')) + await assert.doesNotReject( + verify(capacityDirectory, 'capacity-transition', 'c3') + ) + await assert.doesNotReject(verify(recoveryDirectory, 'recover-forward')) + await assert.doesNotReject(verify(recoveryDirectory, 'fence-source')) + await assert.doesNotReject( + verifyMutationEvidence( + [ + '--directory', + recoveryDirectory, + ...provenance, + '--mutation-mode', + 'execute', + '--source-cell-id', + 'c12', + '--scoped-recovery-source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ) + ) + await assert.doesNotReject( + verifyMutationEvidence( + [ + '--directory', + recoveryDirectory, + ...provenance, + '--mutation-mode', + 'recover-forward', + '--source-cell-id', + 'c12', + '--scoped-recovery-source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ) + ) + await assert.rejects( + verify(recoveryDirectory, 'execute'), + /migration policy does not match/ + ) + await assert.rejects( + verify(strictDirectory, 'recover-forward'), + /migration policy does not match/ + ) + await assert.rejects( + verify(strictDirectory, 'capacity-transition', 'c3'), + /migration policy does not match/ + ) + await assert.rejects( + verify(capacityDirectory, 'capacity-transition', 'c1'), + /capacity cell does not match/ + ) + await assert.rejects( + verifyMutationEvidence( + [ + '--directory', + recoveryDirectory, + ...provenance, + '--mutation-mode', + 'recover-forward', + '--source-cell-id', + 'c9', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ), + /recovery source does not match/ + ) + await assert.rejects( + verifyMutationEvidence( + [ + '--directory', + recoveryDirectory, + ...provenance, + '--mutation-mode', + 'fence-source', + '--source-cell-id', + 'c1', + '--scoped-recovery-source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ), + /scoped recovery evidence is invalid/ + ) + await assert.rejects( + verifyMutationEvidence( + [ + '--directory', + recoveryDirectory, + ...provenance, + '--mutation-mode', + 'execute', + '--source-cell-id', + 'c12', + '--scoped-recovery-source-cell-id', + 'c9', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + fetchImpl, + () => now + ), + /recovery source does not match/ + ) + } finally { + await rm(strictDirectory, { recursive: true, force: true }) + await rm(recoveryDirectory, { recursive: true, force: true }) + await rm(capacityDirectory, { recursive: true, force: true }) + } +}) + +test('workflow reruns restore the prior attempt into one stable incident', async () => { + const workflow = await readFile( + relayWorkflowUrl('monitor-relay-production-job.yml'), + 'utf8' + ) + assert.match(workflow, /INCIDENT_ID: relay-\$\{\{ github\.run_id \}\}-\$\{\{ inputs\.mode \}\}/) + assert.doesNotMatch(workflow, /INCIDENT_ID:.*run_attempt/) + assert.match(workflow, /actions\/download-artifact@v4/) + assert.match(workflow, /verify-restore/) + assert.match(workflow, /RESTART_FLAG=--restart/) + assert.equal(workflow.match(/--capacity-cell-id/g)?.length, 3) + const dispatchWorkflow = await readFile( + relayWorkflowUrl('monitor-relay-production.yml'), + 'utf8' + ) + assert.match(dispatchWorkflow, /- capacity-transition/) + assert.match(dispatchWorkflow, /capacity-cell-id: \$\{\{ inputs\.capacity-cell-id \}\}/) +}) + +test('same-cap and rehome mutations require complete strict dry-run authority', async () => { + for (const name of [ + 'deploy-relay-production-same-cap-job.yml', + 'operate-relay-production-rehome-job.yml' + ]) { + const workflow = await readFile( + relayWorkflowUrl(name), + 'utf8' + ) + assert.match(workflow, /relay-monitor-evidence\.mjs verify-authority/) + assert.match(workflow, /--required-migration-policy strict/) + assert.doesNotMatch(workflow, /relay-monitor-evidence\.mjs verify-restore/) + } +}) + +test('production mutation workflows consume and live-recheck dry-run evidence', async () => { + for (const name of [ + 'deploy-relay-production.yml', + 'deploy-relay-production-multi-target.yml' + ]) { + const workflow = await readFile( + relayWorkflowUrl(name), + 'utf8' + ) + assert.match(workflow, /actions\/download-artifact@v4/) + assert.match(workflow, /verify-mutation/) + assert.match(workflow, /--mutation-mode "\$\{DEPLOY_MODE\}"/) + assert.match(workflow, /--source-cell-id "\$\{SOURCE_CELL_ID\}"/) + assert.match(workflow, /incident:relay-preflight/) + assert.match(workflow, /Reject previously consumed dry-run evidence/) + assert.match(workflow, /actions\/upload-artifact@v4/) + assert.match(workflow, /relay-monitor-consumed-/) + assert.match(workflow, /ORCA_RELAY_ADMIN_ID_TOKEN/) + assert.match(workflow, /github\.ref == 'refs\/heads\/main'/) + assert.ok( + workflow.indexOf('pnpm install --frozen-lockfile') < + workflow.indexOf('id: google-auth') + ) + } +}) + +test('monitor and mutation workflows share the production Cloud SQL rollout lock', async () => { + for (const name of [ + 'monitor-relay-production.yml', + 'deploy-relay-production.yml', + 'deploy-relay-production-multi-target.yml', + 'deploy-relay-production-capacity.yml' + ]) { + const workflow = await readFile( + relayWorkflowUrl(name), + 'utf8' + ) + assert.match(workflow, /group: production-cloud-sql-rollout/) + } +}) + +test('monitor uses a reusable job so exact job_workflow_ref is present', async () => { + const wrapper = await readFile( + relayWorkflowUrl('monitor-relay-production.yml'), + 'utf8' + ) + assert.ok(wrapper.includes(`uses: ./${relayWorkflowPath('monitor-relay-production-job.yml')}`)) + const job = await readFile( + relayWorkflowUrl('monitor-relay-production-job.yml'), + 'utf8' + ) + assert.match(job, /workflow_call:/) + assert.match(job, /environment: production/) +}) diff --git a/cloud/dev/scripts/relay-production-capacity-wave.mjs b/cloud/dev/scripts/relay-production-capacity-wave.mjs new file mode 100644 index 00000000000..e76c264e5ed --- /dev/null +++ b/cloud/dev/scripts/relay-production-capacity-wave.mjs @@ -0,0 +1,172 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { pathToFileURL } from 'node:url' +import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs' + +const APPROVED_CELLS = new Set(PRODUCTION_CAPACITY_CELL_IDS) + +function argumentsByName(argv, allowed) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const argument = argv[index] + const value = argv[index + 1] + if (!argument?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error('capacity wave arguments are invalid') + } + const name = argument.slice(2) + if (!allowed.has(name) || name in values) { + throw new Error(`capacity wave argument --${name} is invalid`) + } + values[name] = value + } + return values +} + +export function parseCapacityWave(waveCellIds, confirmation) { + const cells = waveCellIds.split(',') + if ( + cells.length < 2 || + cells.length > 4 || + cells.some((cell) => !APPROVED_CELLS.has(cell)) || + new Set(cells).size !== cells.length || + cells.join(',') !== waveCellIds + ) { + throw new Error('capacity wave must contain two to four unique approved cells') + } + if (confirmation !== `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`) { + throw new Error('capacity wave confirmation does not match the selected cells') + } + return cells +} + +function exactMembership(membership) { + const keys = ['existingOnly', 'migrationOnly', 'general'] + if (!keys.every((key) => Array.isArray(membership?.[key]))) return false + const cells = keys.flatMap((key) => membership[key]) + return cells.every((cell) => typeof cell === 'string') && new Set(cells).size === cells.length +} + +export function capacityWavePreflightState(state, waveCellIds, waveIndex, targetCellId) { + const cells = parseCapacityWave( + waveCellIds, + `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}` + ) + const index = Number(waveIndex) + const membership = state.expectedSelector?.membership + if ( + !Number.isSafeInteger(index) || + index < 0 || + index >= cells.length || + targetCellId !== cells[index] || + state.schemaVersion !== 4 || + state.environment !== 'production' || + state.preDrainDryRun !== true || + state.migrationPolicy !== 'capacity-transition' || + state.recoverySourceCellId !== null || + state.capacityCellId !== cells[0] || + !Number.isSafeInteger(state.expectedSelector?.generation) || + !exactMembership(membership) || + !cells.every((cell) => membership.general.includes(cell)) || + state.sampleCount < 16 || + state.frozenAt !== null || + typeof state.completedAt !== 'string' + ) { + throw new Error('capacity wave evidence does not match this step') + } + return { + schemaVersion: 4, + environment: 'production', + expectedSelector: { + generation: state.expectedSelector.generation + index * 2, + membership + }, + migrationPolicy: 'capacity-transition', + recoverySourceCellId: null, + capacityCellId: targetCellId + } +} + +export function capacityWaveResumePreflightState(state, waveCellIds, targetCellId) { + const cells = parseCapacityWave( + waveCellIds, + `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}` + ) + const index = cells.indexOf(targetCellId) + const base = capacityWavePreflightState( + state, + waveCellIds, + String(index), + targetCellId + ) + const membership = base.expectedSelector.membership + const general = membership.general.filter((cell) => cell !== targetCellId) + const capacityCellId = general.includes(state.capacityCellId) + ? state.capacityCellId + : cells.find((cell) => general.includes(cell)) + if (!capacityCellId) throw new Error('capacity wave resume has no general evidence cell') + return { + ...base, + expectedSelector: { + generation: base.expectedSelector.generation + 1, + membership: { + existingOnly: membership.existingOnly, + migrationOnly: [...membership.migrationOnly, targetCellId].sort(), + general + } + }, + capacityCellId + } +} + +async function main(argv) { + const [command, ...arguments_] = argv + if (command === 'validate') { + const values = argumentsByName( + arguments_, + new Set(['wave-cell-ids', 'confirmation']) + ) + process.stdout.write(`${JSON.stringify(parseCapacityWave( + values['wave-cell-ids'] ?? '', + values.confirmation ?? '' + ))}\n`) + return + } + if (command === 'build-preflight' || command === 'build-resume-preflight') { + const values = argumentsByName( + arguments_, + new Set([ + 'state-file', + 'wave-cell-ids', + ...(command === 'build-preflight' ? ['wave-index'] : []), + 'target-cell-id', + 'output-file' + ]) + ) + const state = JSON.parse(await readFile(values['state-file'] ?? '', 'utf8')) + const preflight = command === 'build-preflight' + ? capacityWavePreflightState( + state, + values['wave-cell-ids'] ?? '', + values['wave-index'] ?? '', + values['target-cell-id'] ?? '' + ) + : capacityWaveResumePreflightState( + state, + values['wave-cell-ids'] ?? '', + values['target-cell-id'] ?? '' + ) + await writeFile( + values['output-file'] ?? '', + `${JSON.stringify(preflight)}\n`, + { mode: 0o600 } + ) + return + } + throw new Error('capacity wave command is invalid') +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).catch((error) => { + console.error(error instanceof Error ? error.message : 'capacity wave failed') + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/relay-production-capacity-wave.test.mjs b/cloud/dev/scripts/relay-production-capacity-wave.test.mjs new file mode 100644 index 00000000000..2cb8ebfa538 --- /dev/null +++ b/cloud/dev/scripts/relay-production-capacity-wave.test.mjs @@ -0,0 +1,129 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + capacityWavePreflightState, + capacityWaveResumePreflightState, + parseCapacityWave +} from './relay-production-capacity-wave.mjs' + +const wave = [ + 'production-gce-c22', + 'production-gce-c21', + 'production-gce-c20', + 'production-gce-c19' +] + +function evidence(overrides = {}) { + return { + schemaVersion: 4, + environment: 'production', + preDrainDryRun: true, + migrationPolicy: 'capacity-transition', + recoverySourceCellId: null, + capacityCellId: wave[0], + expectedSelector: { + generation: 39, + membership: { + existingOnly: ['production-gce-c1'], + migrationOnly: ['production-gce-c17', 'production-gce-c18'], + general: [...wave, 'production-gce-c23'] + } + }, + sampleCount: 16, + frozenAt: null, + completedAt: '2026-08-11T21:00:00.000Z', + ...overrides + } +} + +test('accepts only exact confirmed waves of two to four approved cells', () => { + for (const cells of [wave.slice(0, 2), wave]) { + const value = cells.join(',') + assert.deepEqual( + parseCapacityWave(value, `RAISE_SELECTED_WAVE_TO_1000 ${value}`), + cells + ) + } + for (const [cells, confirmation] of [ + [[wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}`], + [[...wave, 'production-gce-c16'], `RAISE_SELECTED_WAVE_TO_1000 ${wave.join(',')}`], + [[wave[0], wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},${wave[0]}`], + [[wave[0], 'production-gce-c17'], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},production-gce-c17`], + [[wave[0], ` ${wave[1]}`], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}, ${wave[1]}`], + [wave, 'RAISE_SELECTED_WAVE_TO_1000 production-gce-c22'] + ]) { + assert.throws(() => parseCapacityWave(cells.join(','), confirmation)) + } +}) + +test('derives each continuation preflight from the sealed selector generation', () => { + for (const [index, cell] of wave.entries()) { + const state = capacityWavePreflightState( + evidence(), + wave.join(','), + String(index), + cell + ) + assert.equal(state.expectedSelector.generation, 39 + index * 2) + assert.equal(state.capacityCellId, cell) + assert.deepEqual(state.expectedSelector.membership, evidence().expectedSelector.membership) + } +}) + +test('derives an exact isolated-cell resume state from sealed wave evidence', () => { + const state = capacityWaveResumePreflightState( + evidence(), + wave.join(','), + wave[3] + ) + assert.equal(state.expectedSelector.generation, 46) + assert.equal(state.capacityCellId, wave[0]) + assert.deepEqual(state.expectedSelector.membership, { + existingOnly: ['production-gce-c1'], + migrationOnly: ['production-gce-c17', 'production-gce-c18', wave[3]].sort(), + general: [wave[0], wave[1], wave[2], 'production-gce-c23'] + }) +}) + +test('resume rejects a target outside the exact sealed wave', () => { + assert.throws( + () => capacityWaveResumePreflightState( + evidence(), + wave.join(','), + 'production-gce-c16' + ), + /does not match/ + ) +}) + +test('resume rebinds first-cell evidence to another general wave cell', () => { + const state = capacityWaveResumePreflightState( + evidence(), + wave.join(','), + wave[0] + ) + assert.equal(state.capacityCellId, wave[1]) + assert.equal(state.expectedSelector.generation, 40) + assert.ok(state.expectedSelector.membership.migrationOnly.includes(wave[0])) + assert.ok(state.expectedSelector.membership.general.includes(wave[1])) +}) + +test('rejects reordered, incomplete, frozen, or mismatched wave evidence', () => { + const calls = [ + () => capacityWavePreflightState(evidence(), wave.join(','), '1', wave[0]), + () => capacityWavePreflightState(evidence({ capacityCellId: wave[1] }), wave.join(','), '0', wave[0]), + () => capacityWavePreflightState(evidence({ sampleCount: 15 }), wave.join(','), '0', wave[0]), + () => capacityWavePreflightState(evidence({ frozenAt: '2026-08-11T20:59:00.000Z' }), wave.join(','), '0', wave[0]), + () => capacityWavePreflightState(evidence({ completedAt: null }), wave.join(','), '0', wave[0]), + () => capacityWavePreflightState(evidence({ + expectedSelector: { + ...evidence().expectedSelector, + membership: { + ...evidence().expectedSelector.membership, + general: wave.slice(1) + } + } + }), wave.join(','), '0', wave[0]) + ] + for (const call of calls) assert.throws(call, /does not match/) +}) diff --git a/cloud/dev/scripts/relay-production-capacity-workflow.test.mjs b/cloud/dev/scripts/relay-production-capacity-workflow.test.mjs new file mode 100644 index 00000000000..1c9e3aee5f6 --- /dev/null +++ b/cloud/dev/scripts/relay-production-capacity-workflow.test.mjs @@ -0,0 +1,440 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { readRelayWorkflow, relayWorkflowPath } from './relay-repository.mjs' +import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs' + +const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml') +const workflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml') +const terraform = source('infra/terraform/relay-github-actions.tf') +const production = source('infra/terraform/environments/production.tfvars') +const capacityCells = PRODUCTION_CAPACITY_CELL_IDS + +function source(path) { + return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8') +} + +function resource(type, name) { + const start = terraform.indexOf(`resource "${type}" "${name}"`) + assert.notEqual(start, -1, `${type}.${name} is missing`) + const next = terraform.indexOf('\nresource "', start + 1) + return terraform.slice(start, next === -1 ? undefined : next) +} + +function ordered(...markers) { + let previous = -1 + for (const marker of markers) { + const current = workflow.indexOf(marker) + assert.ok(current > previous, `${marker} is missing or out of order`) + previous = current + } +} + +function mutationConfirmation(mode, targetCellId, confirmation) { + const stepStart = workflow.indexOf(' - name: Require exact mutation confirmation') + const runMarker = ' run: |\n' + const runStart = workflow.indexOf(runMarker, stepStart) + runMarker.length + const runEnd = workflow.indexOf('\n - name:', runStart) + const script = workflow.slice(runStart, runEnd).replace(/^ {10}/gm, '') + return spawnSync('bash', ['-euo', 'pipefail', '-c', script], { + env: { + ...process.env, + DEPLOY_MODE: mode, + TARGET_CELL_ID: targetCellId, + CONFIRMATION: confirmation + } + }).status +} + +function imageCompatibility(activeImage, desiredImage) { + const start = workflow.indexOf(' ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"') + const end = workflow.indexOf('\n CURRENT_CELLS_JSON=', start) + const script = workflow.slice(start, end).replace(/^ {10}/gm, '') + return spawnSync('bash', ['-euo', 'pipefail', '-c', script], { + env: { + ...process.env, + ACTIVE_IMAGE: activeImage, + DESIRED_IMAGE: desiredImage, + DESIRED_IMAGE_DIGEST: desiredImage.split('@').at(-1), + COMPATIBLE_DIRECTOR_IMAGE_DIGEST: 'sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73', + COMPATIBLE_CELL_IMAGE_DIGEST: 'sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f' + } + }).status +} + +test('production capacity mutation is restricted to the exact serving rollout set', () => { + assert.match(workflow, /TARGET_CELL_ID: \$\{\{ inputs\.target-cell-id \}\}/) + const targetInput = dispatchWorkflow.slice( + dispatchWorkflow.indexOf(' target-cell-id:'), + dispatchWorkflow.indexOf(' wave-cell-ids:') + ) + assert.deepEqual( + [...targetInput.matchAll(/^\s+- (production-gce-c\d+)$/gm)].map((match) => match[1]), + capacityCells + ) + assert.match(workflow, new RegExp(`CAPACITY_CELL_IDS: ${capacityCells.join(',')}`)) + for (const cellId of capacityCells) { + assert.match(dispatchWorkflow, new RegExp(`^\\s+- ${cellId}$`, 'm')) + } + assert.match(workflow, /CELL_ORIGIN="https:\/\/\$\{TARGET_HOSTNAME\}\.relay\.onorca\.dev"/) + assert.match(workflow, /echo "TARGET_HOSTNAME=\$\{TARGET_HOSTNAME\}"/) + assert.match(workflow, /\} >> "\$\{GITHUB_ENV\}"/) + assert.match(workflow, /RAISE_SELECTED_CELL_TO_1000/) + assert.match(workflow, /ROLL_BACK_SELECTED_CELL_TO_600/) + assert.match(workflow, /TARGET_HARD_CAP=600[\s\S]*?else[\s\S]*?TARGET_HARD_CAP=1000/) +}) + +test('rollback confirmation is bound to the exact selected cell', () => { + assert.equal( + mutationConfirmation('apply', 'production-gce-c25', 'RAISE_SELECTED_CELL_TO_1000'), + 0 + ) + assert.equal( + mutationConfirmation( + 'rollback', + 'production-gce-c25', + 'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c25' + ), + 0 + ) + assert.notEqual( + mutationConfirmation('rollback', 'production-gce-c25', 'ROLL_BACK_SELECTED_CELL_TO_600'), + 0 + ) + assert.notEqual( + mutationConfirmation( + 'rollback', + 'production-gce-c25', + 'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c26' + ), + 0 + ) +}) + +test('production configuration selects only serving cells for 1,000 and the compatible image', () => { + const cell = (cellId) => production.slice( + production.indexOf(`"${cellId}"`), + production.indexOf('\n }', production.indexOf(`"${cellId}"`)) + ) + for (const cellId of capacityCells) { + assert.match(cell(cellId), /connection_hard_cap\s+= 1000/) + assert.match( + cell(cellId), + /sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563/ + ) + } + for (const cellId of ['production-gce-c17', 'production-gce-c18']) { + assert.match(cell(cellId), /connection_hard_cap\s+= 600/) + assert.doesNotMatch(cell(cellId), /connection_hard_cap\s+= 1000/) + assert.match(cell(cellId), /sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d/) + } + assert.equal(production.match(/connection_hard_cap\s+= 1000/g)?.length, capacityCells.length) + // Asia cells legitimately share this digest, so scope the uniqueness check to the capacity set. + assert.equal( + new Set(capacityCells.map((cellId) => cell(cellId).match(/sha256:[0-9a-f]{64}/)[0])).size, + 1 + ) + assert.match( + workflow, + /COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73/ + ) + assert.match( + workflow, + /test "\$\{ACTIVE_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_DIRECTOR_IMAGE_DIGEST\}"/ + ) + assert.match( + workflow, + /test "\$\{DESIRED_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/ + ) + assert.match(workflow, /\.\[\$cell\]\.connection_hard_cap = \$cap/) + assert.match(workflow, /baseCells:\$baseCells/) + assert.match(workflow, /capacityCellIds:\(\$capacityCellIds \| split\(","\)\)/) + assert.match( + workflow, + /Verify current selected-cell capacity[\s\S]*?TOPOLOGY_PHASE.*predecessor[\s\S]*?CURRENT_CAP=600[\s\S]*?DESIRED_IMAGE_DIGEST.*PREDECESSOR_IMAGE_DIGEST/ + ) +}) + +test('director and cell image compatibility is an exact reviewed pair', () => { + const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@' + const director = `${repository}sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73` + const cell = `${repository}sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f` + const other = `${repository}sha256:${'a'.repeat(64)}` + assert.equal(imageCompatibility(cell, cell), 0) + assert.equal(imageCompatibility(director, cell), 0) + assert.notEqual(imageCompatibility(director, other), 0) + assert.notEqual(imageCompatibility(other, cell), 0) +}) + +// The matching check on google_project_service.required lives with the foundation root, which +// stays in the private repository. + +test('apply consumes fresh evidence before arming mutation cleanup', () => { + for (const marker of [ + 'Require fresh dry-run evidence reference', + 'Verify dry-run artifact before cloud authentication', + 'Reject previously consumed dry-run evidence', + 'Verify fresh dry-run evidence against the live selector', + 'Recheck every live safety signal', + 'Publish the consumed-evidence marker' + ]) { + assert.match(workflow, new RegExp(marker)) + } + assert.match(workflow, /--mutation-mode capacity-transition/) + assert.match(workflow, /--source-cell-id "\$\{TARGET_CELL_ID\}"/) + ordered( + 'Publish the consumed-evidence marker', + 'Arm fail-closed mutation cleanup', + 'Reversibly isolate only the selected cell', + 'Deploy only the reviewed director topology', + 'Plan and apply only the empty selected cell', + 'Restore only the selected cell to general admission', + 'Verify the live general selected cell' + ) +}) + +test('wave apply consumes one proof and runs fail-closed cells sequentially', () => { + assert.match(dispatchWorkflow, /- wave-apply/) + assert.match(dispatchWorkflow, /group: production-cloud-sql-rollout/) + assert.match(dispatchWorkflow, /Validate the exact wave request/) + assert.match(dispatchWorkflow, /Verify wave evidence against the live selector/) + assert.match(dispatchWorkflow, /Require exact 600\/60 predecessor wave cells/) + assert.match( + dispatchWorkflow, + /COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f/ + ) + assert.match( + dispatchWorkflow, + /Require exact 600\/60 predecessor wave cells[\s\S]*?--expected-image-digests \\\n\s+"\$\{PREDECESSOR_IMAGE_DIGEST\},\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/ + ) + assert.match(dispatchWorkflow, /Publish the consumed-evidence marker/) + assert.match( + dispatchWorkflow, + /OUTPUT_DIRECTORY: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/ + ) + assert.match( + dispatchWorkflow, + /path: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/ + ) + assert.doesNotMatch(dispatchWorkflow, /strategy:/) + for (const [index, dependency] of [ + [1, 'wave_gate'], + [2, 'wave_cell_1'], + [3, 'wave_cell_2'], + [4, 'wave_cell_3'] + ]) { + const start = dispatchWorkflow.indexOf(` wave_cell_${index}:`) + const end = dispatchWorkflow.indexOf(`\n wave_cell_${index + 1}:`, start) + const job = dispatchWorkflow.slice(start, end === -1 ? undefined : end) + assert.match(job, new RegExp(`needs: (?:\\[wave_gate, )?${dependency}`)) + assert.match(job, /evidence-mode: continuation/) + assert.match(job, new RegExp(`wave-index: '${index - 1}'`)) + } + assert.match(workflow, /Download this workflow's wave authority/) + assert.match(workflow, /run-id: \$\{\{ github\.run_id \}\}/) + assert.match(workflow, /relay-production-capacity-wave\.mjs build-preflight/) + assert.match(workflow, /Require the exact wave predecessor topology/) + assert.match(workflow, /test "\$\{TOPOLOGY_PHASE\}" = predecessor/) + assert.match(workflow, /Recheck exact wave state and every live safety signal/) + assert.match(workflow, /if test "\$\{WAVE_INDEX\}" != 0; then RETRY_ARGS=\(--retry-freshness\); fi/) + assert.equal(workflow.match(/--retry-freshness/g)?.length, 2) + ordered( + 'Recheck exact wave state and every live safety signal', + 'Arm fail-closed mutation cleanup', + 'Reversibly isolate only the selected cell', + 'Verify the live general selected cell' + ) +}) + +test('Terraform mutation targets only the selected cell and has fail-closed recovery', () => { + assert.match( + workflow, + /google_compute_instance_template\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/ + ) + assert.match( + workflow, + /google_compute_instance_group_manager\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/ + ) + assert.doesNotMatch(workflow, /relay_gce_cell\["production-gce-c26"\]/) + assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/) + assert.match(workflow, /validate-relay-capacity-plan\.mjs/) + assert.match(workflow, /--mode bootstrap-cell/) + assert.match(workflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/) + assert.match(workflow, /failure\(\) && inputs\.mode != 'verify'/) + assert.match(workflow, /test "\$\{MUTATION_STARTED:-false\}" = true \|\| exit 0/) + assert.match(workflow, /--mode isolate/) + assert.doesNotMatch(workflow, /rolling-action restart/) + assert.equal(workflow.match(/manage_artifact_dns=false/g)?.length, 5) + assert.match(workflow, /OFFLINE_ROLLBACK=true/) + assert.match(workflow, /--runtime unavailable/) + assert.equal(workflow.match(/--expected-image-digests/g)?.length, 7) + assert.match(workflow, /PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b/) + assert.match(workflow, /classify-relay-production-capacity-director\.mjs/) + assert.match(workflow, /CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON/) + assert.match(workflow, /if test "\$\{DIRECTOR_READY\}" = true; then exit 0; fi/) + assert.match( + workflow, + /Keep the selected cell isolated after a failed mutation[\s\S]*?--mode isolate[\s\S]*?--mode drain/ + ) + const cleanup = workflow.slice(workflow.indexOf('id: cleanup-auth')) + assert.match(cleanup, /Keep the selected cell isolated after a failed mutation/) + assert.match(cleanup, /steps\.cleanup-auth\.outputs\.id_token/) + assert.doesNotMatch(cleanup, /steps\.deploy-auth\.outputs\.id_token/) + ordered( + 'Reversibly isolate only the selected cell', + 'Drain the selected cell or prove an offline rollback', + 'id: restart-auth-one', + 'Require restart-safe selected-cell activity', + 'id: restart-auth-two', + 'Require extended restart-safe selected-cell activity', + 'Deploy only the reviewed director topology', + 'id: director-transition-auth', + 'Require fail-closed director transition', + 'id: capacity-auth', + 'Plan and apply only the empty selected cell', + 'id: capacity-transition-auth', + 'Verify fresh exact selected-cell heartbeat before admission' + ) + const restartGate = workflow.slice( + workflow.indexOf('id: restart-auth-one'), + workflow.indexOf('Deploy only the reviewed director topology') + ) + assert.equal(restartGate.match(/--timeout-ms 450000/g)?.length, 2) + assert.match(restartGate, /steps\.restart-auth-one\.outputs\.id_token/) + assert.match(restartGate, /steps\.restart-auth-two\.outputs\.id_token/) + assert.match(restartGate, /capacity transition verification timed out:/) + assert.match(workflow, /timeout-minutes: 75/) +}) + +test('wave resume is bound to the failed run and exact isolated selector state', () => { + assert.match(dispatchWorkflow, /- wave-resume/) + assert.match(dispatchWorkflow, /evidence-mode: resume/) + assert.match(dispatchWorkflow, /source-wave-run-id: \$\{\{ inputs\.source-wave-run-id \}\}/) + assert.match(workflow, /Download the failed wave authority for resume/) + assert.match(workflow, /test "\$\{SOURCE_SHA\}" = "\$\{MONITOR_SHA\}"/) + assert.match(workflow, /test "\$\{SOURCE_ATTEMPT\}" = "\$\{EXPECTED_SOURCE_ATTEMPT\}"/) + for (const boundary of [ + '31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13', + '31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10', + '31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8', + 'a917e8e1fc1a2654e8cb81ba39b57733ec56be9c', + '6082e9ca89a918ca51f0c87db003f5e8805b64b7', + 'e59958130c9d9b7a6cd805df2678d08997842c7c', + 'EXPECTED_SOURCE_ATTEMPT=2' + ]) { + assert.match(workflow, new RegExp(boundary)) + } + assert.match(workflow, /\*\) exit 1 ;;/) + assert.match(workflow, /test "\$\{MONITOR_SHA\}" = "\$\{EXPECTED_SHA\}"/) + assert.match(workflow, /\.head_branch == "main"/) + assert.match(workflow, /\.head_repository\.full_name == env\.GITHUB_REPOSITORY/) + assert.ok(workflow.includes(`.path == "${relayWorkflowPath('monitor-relay-production.yml')}"`)) + assert.ok(workflow.includes(`.path == "${relayWorkflowPath('deploy-relay-production-capacity.yml')}"`)) + assert.match(workflow, /build-resume-preflight/) + assert.match(workflow, /RESUME_SELECTED_CELL_TO_1000 \$\{TARGET_CELL_ID\}/) + assert.match( + workflow, + /Recheck exact isolated resume state[\s\S]*?--hard-cap 600[\s\S]*?--admission migration-only[\s\S]*?--draining required/ + ) +}) + +test('GCE capacity identity is exact-workflow and narrowly permissioned', () => { + const provider = resource( + 'google_iam_workload_identity_pool_provider', + 'github_production_relay_capacity' + ) + assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/) + for (const boundary of [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'production'", + 'local.relay_github_workflow_conditions["github_production_relay_capacity"]' + ]) { + assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'))) + } + // The workflow pair itself is pinned in the clause the provider renders, once per accepted + // repository, and each repository supplies its own workflow-ref head. + for (const boundary of [ + "assertion.workflow_ref == '${prefix}${local.github_production_relay_capacity_workflow_file}@refs/heads/main'", + "assertion.job_workflow_ref == '${prefix}${local.github_production_relay_capacity_job_workflow_file}@refs/heads/main'" + ]) { + assert.match(terraform, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'))) + } + const role = resource( + 'google_project_iam_custom_role', + 'github_production_relay_capacity_mutation' + ) + assert.match(role, /compute\.instanceGroupManagers\.update/) + assert.match(role, /compute\.instanceTemplates\.create/) + assert.doesNotMatch( + role, + /compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/ + ) + const state = resource( + 'google_storage_bucket_iam_member', + 'github_production_relay_capacity_state' + ) + assert.match(state, /objects\/terraform\/state\/default\.tfstate/) + assert.match(state, /objects\/terraform\/state\/default\.tflock/) +}) + +test('deploy and capacity identities are used in their intended phases', () => { + const jobStart = workflow.indexOf(' capacity:') + const stepsStart = workflow.indexOf(' steps:', jobStart) + const jobHeader = workflow.slice(jobStart, stepsStart) + assert.deepEqual(jobHeader.match(/^\s+if:.*$/gm), [ + " if: ${{ github.ref == 'refs/heads/main' }}" + ]) + const configurationStart = workflow.indexOf('Require production workflow configuration') + const configurationEnd = workflow.indexOf('- uses: actions/checkout@v4', configurationStart) + assert.ok(configurationStart >= 0) + assert.ok(configurationEnd > configurationStart) + const configurationStep = workflow.slice(configurationStart, configurationEnd) + for (const name of [ + 'GCP_REGION', + 'DEPLOY_WORKLOAD_IDENTITY_PROVIDER', + 'DEPLOY_SERVICE_ACCOUNT', + 'CAPACITY_WORKLOAD_IDENTITY_PROVIDER', + 'CAPACITY_SERVICE_ACCOUNT' + ]) { + assert.match(configurationStep, new RegExp(`test -n "\\$\\{${name}\\}"`)) + } + ordered('Require production workflow configuration', 'id: deploy-auth') + ordered('id: deploy-auth', 'Reversibly isolate only the selected cell', 'id: capacity-auth') + assert.match(workflow, /steps\.deploy-auth\.outputs\.id_token/) + assert.doesNotMatch(workflow, /steps\.capacity-auth\.outputs\.id_token/) + assert.equal( + workflow.match(/steps\.capacity-transition-auth\.outputs\.id_token/g)?.length, + 3 + ) + assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) + assert.match(workflow, /read-relay-production-capacity-identity\.mjs/) + assert.match(workflow, /\(\.directorReady \| type\) == "boolean"/) + assert.match(workflow, /\(\.directorReady \| tostring\)/) +}) + +test('director readiness extraction preserves only JSON booleans', { + skip: spawnSync('jq', ['--version']).status !== 0 +}, () => { + const filter = `if (.directorReady | type) == "boolean" then + (.directorReady | tostring) + else error("invalid directorReady classification") end` + const extract = (input) => spawnSync('jq', ['-er', filter], { + encoding: 'utf8', + input: JSON.stringify(input) + }) + for (const value of [true, false]) { + const result = extract({ directorReady: value }) + assert.equal(result.status, 0) + assert.equal(result.stdout.trim(), String(value)) + } + for (const input of [ + { directorReady: 'true' }, + { directorReady: 'false' }, + { directorReady: null }, + {} + ]) { + assert.notEqual(extract(input).status, 0) + } +}) diff --git a/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs b/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs new file mode 100644 index 00000000000..7e8ea2a05c1 --- /dev/null +++ b/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs @@ -0,0 +1,169 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' +import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs' +import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs' + +async function source(path) { + return await readFile(new URL(`../../${path}`, import.meta.url), 'utf8') +} + +// Why: the shared deploy identity is relay-owned in production and moves with the relay +// extraction, so it needs a name the public repo can carry without touching the app pair. The +// generic names are retired; a workflow that still reads them would silently resolve to nothing. +test('no workflow names the retired generic production deploy identity', async () => { + const files = workflowFiles() + assert.ok(files.length > 20) + const relayReaders = [] + for (const file of files) { + const workflow = readWorkflow(file) + assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER\b/, file) + assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT\b/, file) + if (/PRODUCTION_GCP_RELAY_DEPLOY_/.test(workflow)) relayReaders.push(file) + } + assert.deepEqual(relayReaders.sort(), [ + 'deploy-relay-fence-broker.yml', + 'deploy-relay-production-capacity-job.yml', + 'deploy-relay-production-capacity.yml', + 'deploy-relay-production-director.yml', + 'deploy-relay-production-multi-target.yml', + 'deploy-relay-production-same-cap-job.yml', + 'deploy-relay-production-same-cap.yml', + 'deploy-relay-production.yml', + 'operate-relay-asia-admission.yml', + 'operate-relay-production-rehome-job.yml', + 'publish-relay-production.yml' + ].map((name) => relayWorkflowFile(name)).sort()) +}) + +test('monitor workflow has no shared deploy identity fallback', async () => { + const workflow = readRelayWorkflow('monitor-relay-production-job.yml') + assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT/) + assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/) + assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER/) +}) + +test('relay fencing uses the dedicated requester and private broker', async () => { + const workflow = readRelayWorkflow('deploy-relay-production-multi-target.yml') + assert.match(workflow, /Reject direct-runner Terraform fence aborts/) + assert.match(workflow, /inputs\.mode == 'fence-source'/) + assert.match(workflow, /inputs\.mode == 'abort-fence-source'/) + assert.match(workflow, /inputs\.mode == 'supersede-target'/) + assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT/) + assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_BROKER_URI/) + assert.match(workflow, /Invoke private target-supersession broker/) + assert.match(workflow, /Invoke private source-fence broker/) + assert.match(workflow, /Require exact broker cell contract/) + assert.match(workflow, /Require exact source-fence broker contract/) + assert.match(workflow, /Require private fence-broker environment/) + assert.match( + workflow, + /DEPLOY_MODE\}" = "execute" \|\|\s+"\$\{DEPLOY_MODE\}" = "recover-forward"\) &&\s+"\$\{SOURCE_CELL_ID\}" = "production-gce-c12"/ + ) + assert.match( + workflow, + /--scoped-recovery-source-cell-id\s+production-gce-c3/ + ) + assert.match( + workflow, + /test "\$\{FAILED_TARGET_CELL_ID\}" = "production-gce-c12"/ + ) + assert.match( + workflow, + /test "\$\{REPLACEMENT_TARGET_CELL_ID\}" = "production-gce-c13"/ + ) + assert.match( + workflow, + /test "\$\{TARGET_CELL_IDS\}" = "production-gce-c12,production-gce-c13"/ + ) + assert.match( + workflow, + /test "\$\{TARGET_CELL_IDS\}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"/ + ) + const jobGate = workflow.slice( + workflow.indexOf('jobs:'), + workflow.indexOf('runs-on:') + ) + assert.doesNotMatch(jobGate, /PRODUCTION_GCP_RELAY_FENCE_/) + const brokerStep = workflow.slice( + workflow.indexOf('- name: Invoke private target-supersession broker'), + workflow.indexOf('- name: Preflight or run multi-target evacuation') + ) + assert.match(brokerStep, /steps\.google-fence-broker-auth\.outputs\.id_token/) + assert.doesNotMatch(brokerStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/) + const sourceFenceStep = workflow.slice( + workflow.indexOf('- name: Invoke private source-fence broker'), + workflow.indexOf('- name: Preflight or run multi-target evacuation') + ) + assert.match(sourceFenceStep, /steps\.google-fence-broker-auth\.outputs\.id_token/) + assert.match(sourceFenceStep, /\/v1\/fence-source/) + assert.doesNotMatch(sourceFenceStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/) +}) + +test('Terraform binds dedicated identities to exact OIDC and resource boundaries', async () => { + const terraform = await source('infra/terraform/relay-github-actions.tf') + for (const claim of ['job_workflow_ref', 'workflow_ref', 'ref', 'environment']) { + assert.match(terraform, new RegExp(`assertion\\.${claim}`)) + } + assert.match(terraform, /github_monitor_workflow_file/) + assert.match(terraform, /github_fence_workflow_file/) + assert.match(terraform, /github_production_relay_capacity_job_workflow_file/) + assert.match(terraform, /google_service_account" "github_monitor"/) + assert.match(terraform, /google_service_account" "github_fence"/) + assert.match(terraform, /google_service_account\.github_monitor\[0\]\.member/) + assert.match(terraform, /service_account_id = google_service_account\.github_fence\[0\]\.name/) + assert.match(terraform, /attribute\.relay_ops_identity\/monitor/) + assert.match(terraform, /attribute\.relay_ops_identity\/fence/) + assert.doesNotMatch(terraform, /github_relay_fence_operator/) + assert.doesNotMatch(terraform, /github_terraform_fence_state_writer/) + const broker = await source('infra/terraform/relay-fence-broker.tf') + assert.match(broker, /max_instance_request_concurrency = 1/) + assert.match(broker, /max_instance_count = 1/) + assert.match(broker, /roles\/run\.invoker/) + assert.match(broker, /google_service_account\.github_fence\[0\]\.member/) + assert.doesNotMatch(broker, /allUsers/) + const brokerDeploy = readRelayWorkflow('deploy-relay-fence-broker.yml') + assert.match(brokerDeploy, /sha-\$\{GITHUB_SHA\}/) + assert.match(brokerDeploy, /gcloud run services update/) + assert.match(brokerDeploy, /\.status\.traffic/) + assert.doesNotMatch(brokerDeploy, /latestReadyRevisionName/) + assert.doesNotMatch(brokerDeploy, /--set-env-vars/) +}) + +test('Terraform exposes the audited production environment values', async () => { + const outputs = await source('infra/terraform/outputs.tf') + for (const output of [ + 'github_relay_monitor_workload_identity_provider', + 'github_relay_monitor_service_account', + 'github_relay_fence_workload_identity_provider', + 'github_relay_fence_service_account' + ]) { + assert.match(outputs, new RegExp(`output "${output}"`)) + } +}) + +test('production mutations pass the minted admin token to live preflight', async () => { + const workflow = readRelayWorkflow('deploy-relay-production.yml') + const recheck = workflow.slice( + workflow.indexOf('- name: Recheck all live safety signals'), + workflow.indexOf('- name: Create single-use dry-run marker') + ) + assert.match( + recheck, + /ORCA_RELAY_ADMIN_ID_TOKEN: \$\{\{ steps\.google-auth\.outputs\.id_token \}\}/ + ) + const multiTarget = readRelayWorkflow('deploy-relay-production-multi-target.yml') + const multiTargetRecheck = multiTarget.slice( + multiTarget.indexOf('- name: Recheck all live safety signals'), + multiTarget.indexOf('- name: Create single-use dry-run marker') + ) + assert.match(multiTargetRecheck, /steps\.google-auth\.outputs\.id_token/) + assert.match(multiTargetRecheck, /inputs\.mode != 'supersede-target'/) +}) + +test('fence broker pins the production-proven Terraform planner', async () => { + const dockerfile = await source('apps/relay-fence-broker/Dockerfile') + assert.match(dockerfile, /FROM hashicorp\/terraform:1\.15\.8 AS terraform/) +}) diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.mjs new file mode 100644 index 00000000000..e391c4c4381 --- /dev/null +++ b/cloud/dev/scripts/relay-production-same-cap-wave.mjs @@ -0,0 +1,161 @@ +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +export const SAME_CAP_CELLS = [ + 'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10', + 'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16', + 'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22', + 'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26', + 'production-gce-c27', 'production-gce-c28', 'production-gce-c29' +] + +function digest(value, name) { + if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`) + return value +} + +function cells(value) { + const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean) + if ( + parsed.length < 1 || + parsed.length > 4 || + new Set(parsed).size !== parsed.length || + parsed.some((cell) => !SAME_CAP_CELLS.includes(cell)) + ) throw new Error('same-cap wave cells are invalid') + return parsed +} + +export function validateSameCapWave(input) { + if (!['verify', 'canary-apply', 'batch-apply', 'rollback'].includes(input.mode)) { + throw new Error('same-cap wave mode is invalid') + } + const selected = cells(input.cellIds) + const targetDigest = digest(input.targetDigest, 'target digest') + const rollbackDigest = digest(input.rollbackDigest, 'rollback digest') + if (targetDigest === rollbackDigest) throw new Error('target and rollback digests must differ') + if (input.mode === 'canary-apply' && selected.length !== 1) { + throw new Error('canary mode requires exactly one cell') + } + if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) { + throw new Error('batch mode requires two to four cells') + } + // Later waves expect the selector to advance by exactly 2 per predecessor, + // which a resumed rollback cell (isolate skipped, +1) violates. + if (input.mode === 'rollback' && selected.length !== 1) { + throw new Error('rollback mode requires exactly one cell') + } + const mutation = input.mode !== 'verify' + const expectedConfirmation = input.mode === 'rollback' + ? `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${selected.join(',')}` + : `ROLL_RELAY_SAME_CAP ${targetDigest} ${selected.join(',')}` + if (mutation && input.confirmation !== expectedConfirmation) { + throw new Error('same-cap confirmation does not match the exact digest and cells') + } + if (!mutation && input.confirmation) throw new Error('verify does not accept confirmation') + if (input.mode === 'batch-apply' && !/^[1-9][0-9]*$/.test(input.canaryRunId ?? '')) { + throw new Error('batch mode requires a canary run ID') + } + if (input.mode !== 'batch-apply' && input.canaryRunId) { + throw new Error('only batch mode accepts a canary run ID') + } + return { cells: selected, targetDigest, rollbackDigest } +} + +export function canaryAuthority(input) { + const wave = validateSameCapWave({ ...input, mode: 'canary-apply', canaryRunId: '' }) + if (!/^[0-9a-f]{40}$/.test(input.commitSha ?? '')) throw new Error('commit SHA is invalid') + if (!/^[1-9][0-9]*$/.test(input.runId ?? '')) throw new Error('run ID is invalid') + const selectorGeneration = Number(input.selectorGeneration) + const rehomeGeneration = Number(input.rehomeGeneration) + if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) { + throw new Error('selector generation is invalid') + } + if (!Number.isSafeInteger(rehomeGeneration) || rehomeGeneration < 0) { + throw new Error('rehome generation is invalid') + } + return { + v: 1, + commitSha: input.commitSha, + runId: input.runId, + cellId: wave.cells[0], + targetDigest: wave.targetDigest, + rollbackDigest: wave.rollbackDigest, + selectorGeneration: selectorGeneration + 2, + rehomeGeneration + } +} + +export function verifyCanaryAuthority(authority, expected) { + if ( + authority?.v !== 1 || + authority.commitSha !== expected.commitSha || + authority.runId !== expected.runId || + authority.targetDigest !== expected.targetDigest || + authority.rollbackDigest !== expected.rollbackDigest || + authority.selectorGeneration !== Number(expected.selectorGeneration) || + authority.rehomeGeneration !== Number(expected.rehomeGeneration) || + !SAME_CAP_CELLS.includes(authority.cellId) + ) throw new Error('canary authority does not match this batch') + return authority +} + +function values(argv) { + const result = {} + for (let index = 0; index < argv.length; index += 2) { + if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) { + throw new Error('invalid arguments') + } + result[argv[index].slice(2)] = argv[index + 1] + } + return result +} + +export function main(argv = process.argv.slice(2)) { + const command = argv.shift() + const input = values(argv) + if (command === 'validate') { + const wave = validateSameCapWave({ + mode: input.mode, + cellIds: input['cell-ids'], + targetDigest: input['target-digest'], + rollbackDigest: input['rollback-digest'], + confirmation: input.confirmation, + canaryRunId: input['canary-run-id'] + }) + process.stdout.write(`${JSON.stringify(wave.cells)}\n`) + return + } + if (command === 'create-canary') { + process.stdout.write(`${JSON.stringify(canaryAuthority({ + mode: 'canary-apply', + cellIds: input['cell-id'], + targetDigest: input['target-digest'], + rollbackDigest: input['rollback-digest'], + confirmation: input.confirmation, + commitSha: input['commit-sha'], + runId: input['run-id'], + selectorGeneration: input['selector-generation'], + rehomeGeneration: input['rehome-generation'] + }))}\n`) + return + } + if (command === 'verify-canary') { + verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), { + commitSha: input['commit-sha'], + runId: input['run-id'], + targetDigest: input['target-digest'], + rollbackDigest: input['rollback-digest'], + selectorGeneration: input['selector-generation'], + rehomeGeneration: input['rehome-generation'] + }) + return + } + throw new Error('unknown same-cap wave command') +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + try { main() } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } +} diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs new file mode 100644 index 00000000000..0b45ae85a99 --- /dev/null +++ b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs @@ -0,0 +1,106 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + canaryAuthority, + validateSameCapWave, + verifyCanaryAuthority +} from './relay-production-same-cap-wave.mjs' + +const targetDigest = `sha256:${'a'.repeat(64)}` +const rollbackDigest = `sha256:${'b'.repeat(64)}` + +test('requires one canary or a bounded reviewed batch', () => { + assert.deepEqual(validateSameCapWave({ + mode: 'canary-apply', + cellIds: 'production-gce-c7', + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7` + }).cells, ['production-gce-c7']) + assert.throws(() => validateSameCapWave({ + mode: 'canary-apply', + cellIds: 'production-gce-c7,production-gce-c8', + targetDigest, + rollbackDigest, + confirmation: 'wrong' + }), /canary/) + assert.deepEqual(validateSameCapWave({ + mode: 'batch-apply', + cellIds: 'production-gce-c8,production-gce-c9', + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c8,production-gce-c9`, + canaryRunId: '42' + }).cells, ['production-gce-c8', 'production-gce-c9']) + assert.deepEqual(validateSameCapWave({ + mode: 'canary-apply', + cellIds: 'production-gce-c28', + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c28` + }).cells, ['production-gce-c28']) + assert.throws(() => validateSameCapWave({ + mode: 'canary-apply', + cellIds: 'production-gce-c30', + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c30` + }), /cells/) +}) + +test('binds rollback confirmation to the exact digest and ordered cells', () => { + assert.throws(() => validateSameCapWave({ + mode: 'rollback', + cellIds: 'production-gce-c7', + targetDigest, + rollbackDigest, + confirmation: `ROLL_BACK_RELAY_SAME_CAP ${targetDigest} production-gce-c7` + }), /confirmation/) +}) + +test('rollback rolls exactly one cell so later waves stay unreachable', () => { + const cellIds = 'production-gce-c7,production-gce-c8' + assert.throws(() => validateSameCapWave({ + mode: 'rollback', + cellIds, + targetDigest, + rollbackDigest, + confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${cellIds}` + }), /rollback mode requires exactly one cell/) + assert.deepEqual(validateSameCapWave({ + mode: 'rollback', + cellIds: 'production-gce-c7', + targetDigest, + rollbackDigest, + confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c7` + }).cells, ['production-gce-c7']) +}) + +test('seals and verifies canary authority for later batches', () => { + const authority = canaryAuthority({ + cellIds: 'production-gce-c7', + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, + commitSha: 'c'.repeat(40), + runId: '42', + selectorGeneration: '11', + rehomeGeneration: '4' + }) + assert.equal(verifyCanaryAuthority(authority, { + commitSha: 'c'.repeat(40), + runId: '42', + targetDigest, + rollbackDigest, + selectorGeneration: '13', + rehomeGeneration: '4' + }).cellId, 'production-gce-c7') + assert.throws(() => verifyCanaryAuthority(authority, { + commitSha: 'd'.repeat(40), + runId: '42', + targetDigest, + rollbackDigest, + selectorGeneration: '11', + rehomeGeneration: '4' + }), /does not match/) +}) diff --git a/cloud/dev/scripts/relay-public-workflow-contract.test.mjs b/cloud/dev/scripts/relay-public-workflow-contract.test.mjs new file mode 100644 index 00000000000..56393d07bd1 --- /dev/null +++ b/cloud/dev/scripts/relay-public-workflow-contract.test.mjs @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + isEntrypoint, + jobIf, + jobNeeds, + jobs, + readWorkflow, + workflowFiles +} from './cloud-sql-rollout-lock-census.mjs' +import { relayWorkflowFile } from './relay-repository.mjs' + +// Why: this repository publishes the relay's operate surface next to the desktop app. Three +// invariants make that safe, and each of them is one careless edit away from being lost. +const OPERATIONS_GATE = "vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'" + +// Cloud Verify is the only cloud workflow that must run on every pull request. +const UNGATED = relayWorkflowFile('verify.yml') + +const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED) + +test('the copy carries every relay workflow', () => { + assert.equal(relayWorkflows().length, 24) +}) + +// Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming +// one of these silently breaks the recovery chain with no failing run to notice. +test('the recovery chain keeps the display names it is matched by', () => { + const names = Object.fromEntries( + ['prove-relay-staging-capacity.yml', 'recover-relay-staging-c4-image.yml', 'requeue-relay-staging-c4-recovery.yml'].map( + (name) => [name, /^name: (.+)$/m.exec(readWorkflow(relayWorkflowFile(name)))?.[1]] + ) + ) + assert.deepEqual(names, { + 'prove-relay-staging-capacity.yml': 'Prove Relay Staging Capacity', + 'recover-relay-staging-c4-image.yml': 'Recover Relay Staging C4 Image', + 'requeue-relay-staging-c4-recovery.yml': 'Requeue Relay Staging C4 Recovery' + }) + const recover = readWorkflow(relayWorkflowFile('recover-relay-staging-c4-image.yml')) + const requeue = readWorkflow(relayWorkflowFile('requeue-relay-staging-c4-recovery.yml')) + assert.ok(recover.includes(`workflows: [${names['prove-relay-staging-capacity.yml']}]`)) + assert.ok(requeue.includes(`workflows: [${names['recover-relay-staging-c4-image.yml']}]`)) +}) + +// Why: this repository holds none of the GCP credentials these workflows would need. Every one +// authenticates through Workload Identity read from a variable, so any repository secret other +// than the automatic token would be a credential the owner has to store here. +test('no cloud workflow reads a repository secret', () => { + for (const file of workflowFiles()) { + for (const [, name] of readWorkflow(file).matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) { + assert.equal(name, 'GITHUB_TOKEN', `${file} reads secrets.${name}`) + } + } +}) + +// Why: the operations gate is what makes the whole surface inert until the owner enables it. A +// job that can start without a gated dependency would run the moment someone dispatches it. +test('every job that can start on its own is gated on the operations variable', () => { + const reachable = [] + for (const file of relayWorkflows()) { + const text = readWorkflow(file) + if (!isEntrypoint(text)) continue + for (const job of jobs(text)) { + if (jobNeeds(job.text).length > 0) continue + reachable.push(`${file}:${job.id}`) + assert.ok(jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} is not gated`) + } + } + assert.ok(reachable.length >= 20, `only ${reachable.length} root jobs were checked`) +}) + +// Why: reusable jobs inherit the caller's gate. Gating them again would be dead configuration +// that reads as protection, and every caller is already checked above. +test('reusable workflows carry no gate of their own', () => { + for (const file of relayWorkflows()) { + const text = readWorkflow(file) + if (isEntrypoint(text)) continue + for (const job of jobs(text)) { + assert.ok(!jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} regates a reusable job`) + } + } +}) diff --git a/cloud/dev/scripts/relay-recovery-wave-gate.mjs b/cloud/dev/scripts/relay-recovery-wave-gate.mjs new file mode 100644 index 00000000000..08d53d3680f --- /dev/null +++ b/cloud/dev/scripts/relay-recovery-wave-gate.mjs @@ -0,0 +1,517 @@ +const EXPECTED_KEYS = { + report: ['schemaVersion', 'environment', 'load', 'outcomes'], + environment: [ + 'projectId', + 'directorOrigin', + 'databaseVcpu', + 'databasePoolMax', + 'publicConcurrentMax', + 'resolvePrioritySlots', + 'directorMinInstances', + 'directorMaxInstances', + 'cloudRunConcurrency', + 'rolloutOldPublicConcurrentMax', + 'rolloutOldResolvePrioritySlots', + 'rolloutNewPublicConcurrentMax', + 'rolloutNewResolvePrioritySlots' + ], + load: [ + 'drainingDesktops', + 'backgroundRequestsPerMinute', + 'backgroundAssignmentRequestsPerMinute', + 'backgroundAssignment503PerMinute', + 'targetConnectionCap', + 'targetCells' + ], + targetCell: ['cellId', 'peakConnections', 'recoveredControls'], + outcomes: [ + 'migrationExpirations', + 'migrationAborts', + 'transactionRetryExhaustions', + 'keyProvenTargetRegistrations', + 'oldestMigrationLeaseRemainingAtDrainMs', + 'targetRegistrationDurationMs', + 'assignmentSuccessesPerMinuteBaseline', + 'assignmentSuccessesPerMinuteMinimum', + 'eligibleResolveRequests', + 'resolve2xx', + 'resolveOverload', + 'readinessChecks', + 'readinessFailures', + 'maintenanceOperations', + 'maintenanceFailures', + 'directorPeakInstances', + 'rolloutOverlapPeakInstances', + 'rolloutOverlapPeakPublicOperations', + 'rolloutOverlapEligibleResolveRequests', + 'rolloutOverlapResolve2xx', + 'rolloutOverlapResolveOverload', + 'rolloutOverlapReadinessFailures', + 'rolloutOverlapPoolWaitP95Ms', + 'rolloutOverlapDatabaseCpuPercentMax', + 'poolWaitP95Ms', + 'poolWaitMaxMs', + 'databaseCpuPercentP95', + 'databaseCpuPercentMax', + 'recoveryDurationMs' + ] +} + +const LIMITS = { + drainingDesktopsMin: 760, + drainingDesktopsMax: 840, + backgroundRequestsPerMinuteMin: 10_450, + backgroundRequestsPerMinuteMax: 11_550, + backgroundAssignment503PerMinuteMin: 8_500, + backgroundAssignment503PerMinuteMax: 10_500, + targetCellCount: 2, + targetConnectionCap: 600, + databaseVcpu: 2, + databasePoolMax: 3, + publicConcurrentMax: 2, + resolvePrioritySlots: 1, + directorMinInstances: 1, + directorMaxInstances: 2, + directorPeakInstances: 2, + rolloutOverlapPeakInstances: 4, + rolloutOverlapPeakPublicOperations: 8, + cloudRunConcurrency: 80, + rolloutOldPublicConcurrentMax: 2, + rolloutOldResolvePrioritySlots: 0, + rolloutNewPublicConcurrentMax: 2, + rolloutNewResolvePrioritySlots: 1, + assignmentThroughputRetentionMin: 0.9, + resolveSuccessRateMin: 0.95, + resolveOverloadRateMaxExclusive: 0.01, + poolWaitP95MsMaxExclusive: 500, + poolWaitMaxMsMaxExclusive: 5_000, + databaseCpuPercentP95MaxExclusive: 70, + databaseCpuPercentMaxMaxExclusive: 85, + oldestMigrationLeaseRemainingAtDrainMsMin: 10 * 60_000, + targetRegistrationDurationMsMax: 5 * 60_000, + recoveryDurationMsMax: 14 * 60_000 +} + +export function evaluateRecoveryWaveReport(input) { + const report = parseReport(input) + const recoveredControls = report.load.targetCells.reduce( + (total, cell) => total + cell.recoveredControls, + 0 + ) + const peakTargetConnections = Math.max( + ...report.load.targetCells.map((cell) => cell.peakConnections) + ) + const assignmentThroughputRetention = ratio( + report.outcomes.assignmentSuccessesPerMinuteMinimum, + report.outcomes.assignmentSuccessesPerMinuteBaseline + ) + const resolveSuccessRate = ratio( + report.outcomes.resolve2xx, + report.outcomes.eligibleResolveRequests + ) + const resolveOverloadRate = ratio( + report.outcomes.resolveOverload, + report.outcomes.eligibleResolveRequests + ) + const rolloutOverlapResolveSuccessRate = ratio( + report.outcomes.rolloutOverlapResolve2xx, + report.outcomes.rolloutOverlapEligibleResolveRequests + ) + const rolloutOverlapResolveOverloadRate = ratio( + report.outcomes.rolloutOverlapResolveOverload, + report.outcomes.rolloutOverlapEligibleResolveRequests + ) + const thresholds = [ + equal('database_vcpu', report.environment.databaseVcpu, LIMITS.databaseVcpu), + equal('database_pool_max', report.environment.databasePoolMax, LIMITS.databasePoolMax), + equal( + 'public_concurrent_max', + report.environment.publicConcurrentMax, + LIMITS.publicConcurrentMax + ), + equal( + 'resolve_priority_slots', + report.environment.resolvePrioritySlots, + LIMITS.resolvePrioritySlots + ), + equal( + 'director_min_instances', + report.environment.directorMinInstances, + LIMITS.directorMinInstances + ), + equal( + 'director_max_instances', + report.environment.directorMaxInstances, + LIMITS.directorMaxInstances + ), + equal( + 'cloud_run_concurrency', + report.environment.cloudRunConcurrency, + LIMITS.cloudRunConcurrency + ), + equal( + 'rollout_old_public_concurrent_max', + report.environment.rolloutOldPublicConcurrentMax, + LIMITS.rolloutOldPublicConcurrentMax + ), + equal( + 'rollout_old_resolve_priority_slots', + report.environment.rolloutOldResolvePrioritySlots, + LIMITS.rolloutOldResolvePrioritySlots + ), + equal( + 'rollout_new_public_concurrent_max', + report.environment.rolloutNewPublicConcurrentMax, + LIMITS.rolloutNewPublicConcurrentMax + ), + equal( + 'rollout_new_resolve_priority_slots', + report.environment.rolloutNewResolvePrioritySlots, + LIMITS.rolloutNewResolvePrioritySlots + ), + between( + 'draining_desktops', + report.load.drainingDesktops, + LIMITS.drainingDesktopsMin, + LIMITS.drainingDesktopsMax + ), + between( + 'background_requests_per_minute', + report.load.backgroundRequestsPerMinute, + LIMITS.backgroundRequestsPerMinuteMin, + LIMITS.backgroundRequestsPerMinuteMax + ), + between( + 'background_assignment_requests_per_minute', + report.load.backgroundAssignmentRequestsPerMinute, + LIMITS.backgroundRequestsPerMinuteMin, + LIMITS.backgroundRequestsPerMinuteMax + ), + between( + 'background_assignment_503_per_minute', + report.load.backgroundAssignment503PerMinute, + LIMITS.backgroundAssignment503PerMinuteMin, + LIMITS.backgroundAssignment503PerMinuteMax + ), + atMost( + 'background_assignment_requests_within_total', + report.load.backgroundAssignmentRequestsPerMinute, + report.load.backgroundRequestsPerMinute + ), + atMost( + 'background_assignment_503_within_assignments', + report.load.backgroundAssignment503PerMinute, + report.load.backgroundAssignmentRequestsPerMinute + ), + equal('target_cell_count', report.load.targetCells.length, LIMITS.targetCellCount), + equal( + 'target_connection_cap', + report.load.targetConnectionCap, + LIMITS.targetConnectionCap + ), + atMost( + 'peak_target_connections', + peakTargetConnections, + report.load.targetConnectionCap + ), + equal('recovered_controls', recoveredControls, report.load.drainingDesktops), + equal('migration_expirations', report.outcomes.migrationExpirations, 0), + equal('migration_aborts', report.outcomes.migrationAborts, 0), + equal('transaction_retry_exhaustions', report.outcomes.transactionRetryExhaustions, 0), + equal( + 'key_proven_target_registrations', + report.outcomes.keyProvenTargetRegistrations, + report.load.drainingDesktops + ), + atLeast( + 'oldest_migration_lease_remaining_at_drain_ms', + report.outcomes.oldestMigrationLeaseRemainingAtDrainMs, + LIMITS.oldestMigrationLeaseRemainingAtDrainMsMin + ), + atMost( + 'target_registration_duration_ms', + report.outcomes.targetRegistrationDurationMs, + LIMITS.targetRegistrationDurationMsMax + ), + atLeast( + 'assignment_throughput_retention', + assignmentThroughputRetention, + LIMITS.assignmentThroughputRetentionMin + ), + atLeast('resolve_success_rate', resolveSuccessRate, LIMITS.resolveSuccessRateMin), + lessThan( + 'resolve_overload_rate', + resolveOverloadRate, + LIMITS.resolveOverloadRateMaxExclusive + ), + atLeast('eligible_resolve_requests', report.outcomes.eligibleResolveRequests, 100), + equal('readiness_failures', report.outcomes.readinessFailures, 0), + atLeast('readiness_checks', report.outcomes.readinessChecks, 1), + equal('maintenance_failures', report.outcomes.maintenanceFailures, 0), + atLeast('maintenance_operations', report.outcomes.maintenanceOperations, 1), + equal( + 'director_peak_instances', + report.outcomes.directorPeakInstances, + LIMITS.directorPeakInstances + ), + equal( + 'rollout_overlap_peak_instances', + report.outcomes.rolloutOverlapPeakInstances, + LIMITS.rolloutOverlapPeakInstances + ), + equal( + 'rollout_overlap_peak_public_operations', + report.outcomes.rolloutOverlapPeakPublicOperations, + LIMITS.rolloutOverlapPeakPublicOperations + ), + atLeast( + 'rollout_overlap_eligible_resolve_requests', + report.outcomes.rolloutOverlapEligibleResolveRequests, + 100 + ), + atLeast( + 'rollout_overlap_resolve_success_rate', + rolloutOverlapResolveSuccessRate, + LIMITS.resolveSuccessRateMin + ), + lessThan( + 'rollout_overlap_resolve_overload_rate', + rolloutOverlapResolveOverloadRate, + LIMITS.resolveOverloadRateMaxExclusive + ), + equal( + 'rollout_overlap_readiness_failures', + report.outcomes.rolloutOverlapReadinessFailures, + 0 + ), + lessThan( + 'rollout_overlap_pool_wait_p95_ms', + report.outcomes.rolloutOverlapPoolWaitP95Ms, + LIMITS.poolWaitP95MsMaxExclusive + ), + lessThan( + 'rollout_overlap_database_cpu_percent_max', + report.outcomes.rolloutOverlapDatabaseCpuPercentMax, + LIMITS.databaseCpuPercentMaxMaxExclusive + ), + lessThan( + 'pool_wait_p95_ms', + report.outcomes.poolWaitP95Ms, + LIMITS.poolWaitP95MsMaxExclusive + ), + lessThan( + 'pool_wait_max_ms', + report.outcomes.poolWaitMaxMs, + LIMITS.poolWaitMaxMsMaxExclusive + ), + lessThan( + 'database_cpu_percent_p95', + report.outcomes.databaseCpuPercentP95, + LIMITS.databaseCpuPercentP95MaxExclusive + ), + lessThan( + 'database_cpu_percent_max', + report.outcomes.databaseCpuPercentMax, + LIMITS.databaseCpuPercentMaxMaxExclusive + ), + atMost( + 'recovery_duration_ms', + report.outcomes.recoveryDurationMs, + LIMITS.recoveryDurationMsMax + ) + ] + return { + schemaVersion: 1, + status: thresholds.every(({ pass }) => pass) ? 'PASS' : 'FAIL', + environment: { + projectId: report.environment.projectId, + directorOrigin: report.environment.directorOrigin + }, + metrics: { + recoveredControls, + peakTargetConnections, + assignmentThroughputRetention, + resolveSuccessRate, + resolveOverloadRate, + rolloutOverlapResolveSuccessRate, + rolloutOverlapResolveOverloadRate + }, + thresholds + } +} + +function parseReport(input) { + const report = strictObject(input, EXPECTED_KEYS.report, 'report') + if (report.schemaVersion !== 1) throw new Error('unsupported report schemaVersion') + const environment = strictObject( + report.environment, + EXPECTED_KEYS.environment, + 'environment' + ) + assertSafeEnvironment(environment) + const load = strictObject(report.load, EXPECTED_KEYS.load, 'load') + if (!Array.isArray(load.targetCells)) throw new Error('load.targetCells must be an array') + const targetCells = load.targetCells.map((value, index) => { + const cell = strictObject(value, EXPECTED_KEYS.targetCell, `load.targetCells[${index}]`) + if (!/^[a-z0-9-]{1,128}$/.test(cell.cellId)) throw new Error('target cellId is invalid') + return { + cellId: cell.cellId, + peakConnections: nonnegativeNumber(cell.peakConnections, 'peakConnections'), + recoveredControls: nonnegativeNumber(cell.recoveredControls, 'recoveredControls') + } + }) + if (new Set(targetCells.map(({ cellId }) => cellId)).size !== targetCells.length) { + throw new Error('target cell IDs must be unique') + } + const outcomes = strictObject(report.outcomes, EXPECTED_KEYS.outcomes, 'outcomes') + return { + schemaVersion: 1, + environment: { + projectId: environment.projectId, + directorOrigin: environment.directorOrigin, + databaseVcpu: positiveNumber(environment.databaseVcpu, 'databaseVcpu'), + databasePoolMax: positiveNumber(environment.databasePoolMax, 'databasePoolMax'), + publicConcurrentMax: positiveNumber( + environment.publicConcurrentMax, + 'publicConcurrentMax' + ), + resolvePrioritySlots: positiveNumber( + environment.resolvePrioritySlots, + 'resolvePrioritySlots' + ), + directorMinInstances: positiveNumber( + environment.directorMinInstances, + 'directorMinInstances' + ), + directorMaxInstances: positiveNumber( + environment.directorMaxInstances, + 'directorMaxInstances' + ), + cloudRunConcurrency: positiveNumber( + environment.cloudRunConcurrency, + 'cloudRunConcurrency' + ), + rolloutOldPublicConcurrentMax: positiveNumber( + environment.rolloutOldPublicConcurrentMax, + 'rolloutOldPublicConcurrentMax' + ), + rolloutOldResolvePrioritySlots: nonnegativeNumber( + environment.rolloutOldResolvePrioritySlots, + 'rolloutOldResolvePrioritySlots' + ), + rolloutNewPublicConcurrentMax: positiveNumber( + environment.rolloutNewPublicConcurrentMax, + 'rolloutNewPublicConcurrentMax' + ), + rolloutNewResolvePrioritySlots: positiveNumber( + environment.rolloutNewResolvePrioritySlots, + 'rolloutNewResolvePrioritySlots' + ) + }, + load: { + drainingDesktops: positiveNumber(load.drainingDesktops, 'drainingDesktops'), + backgroundRequestsPerMinute: positiveNumber( + load.backgroundRequestsPerMinute, + 'backgroundRequestsPerMinute' + ), + backgroundAssignmentRequestsPerMinute: positiveNumber( + load.backgroundAssignmentRequestsPerMinute, + 'backgroundAssignmentRequestsPerMinute' + ), + backgroundAssignment503PerMinute: nonnegativeNumber( + load.backgroundAssignment503PerMinute, + 'backgroundAssignment503PerMinute' + ), + targetConnectionCap: positiveNumber(load.targetConnectionCap, 'targetConnectionCap'), + targetCells + }, + outcomes: Object.fromEntries( + EXPECTED_KEYS.outcomes.map((key) => [key, nonnegativeNumber(outcomes[key], `outcomes.${key}`)]) + ) + } +} + +function assertSafeEnvironment(environment) { + if (typeof environment.projectId !== 'string') throw new Error('projectId must be a string') + if ( + environment.projectId !== 'local' && + !environment.projectId.endsWith('-staging') && + !environment.projectId.endsWith('-test') + ) { + throw new Error('recovery-wave reports must come from an isolated non-production project') + } + if (typeof environment.directorOrigin !== 'string') { + throw new Error('directorOrigin must be a string') + } + const origin = new URL(environment.directorOrigin) + const loopback = ['localhost', '127.0.0.1', '::1', '[::1]'].includes(origin.hostname) + const isolatedHost = + loopback || origin.hostname.endsWith('.test') || origin.hostname.includes('staging') + if ( + origin.origin !== environment.directorOrigin || + origin.pathname !== '/' || + (!loopback && origin.protocol !== 'https:') || + !isolatedHost + ) { + throw new Error('directorOrigin must identify a canonical isolated non-production origin') + } +} + +function strictObject(value, keys, name) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${name} must be an object`) + } + const actual = Object.keys(value).sort() + const expected = [...keys].sort() + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new Error(`${name} has unexpected or missing fields`) + } + return value +} + +function positiveNumber(value, name) { + const number = nonnegativeNumber(value, name) + if (number <= 0) throw new Error(`${name} must be positive`) + return number +} + +function nonnegativeNumber(value, name) { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`${name} must be a finite nonnegative number`) + } + return value +} + +function ratio(numerator, denominator) { + return denominator === 0 ? 0 : numerator / denominator +} + +function equal(name, observed, limit) { + return threshold(name, observed, '==', limit, observed === limit) +} + +function atLeast(name, observed, limit) { + return threshold(name, observed, '>=', limit, observed >= limit) +} + +function atMost(name, observed, limit) { + return threshold(name, observed, '<=', limit, observed <= limit) +} + +function lessThan(name, observed, limit) { + return threshold(name, observed, '<', limit, observed < limit) +} + +function between(name, observed, minimum, maximum) { + return threshold( + name, + observed, + 'between_inclusive', + [minimum, maximum], + observed >= minimum && observed <= maximum + ) +} + +function threshold(name, observed, operator, limit, pass) { + return { name, observed, operator, limit, pass } +} diff --git a/cloud/dev/scripts/relay-recovery-wave-gate.test.mjs b/cloud/dev/scripts/relay-recovery-wave-gate.test.mjs new file mode 100644 index 00000000000..4ac1de83e72 --- /dev/null +++ b/cloud/dev/scripts/relay-recovery-wave-gate.test.mjs @@ -0,0 +1,214 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { evaluateRecoveryWaveReport } from './relay-recovery-wave-gate.mjs' + +test('passes a complete isolated production-shaped recovery report', () => { + const result = evaluateRecoveryWaveReport(passingReport()) + + assert.equal(result.status, 'PASS') + assert.equal(result.metrics.recoveredControls, 800) + assert.equal(result.metrics.peakTargetConnections, 425) + assert.equal(result.metrics.resolveSuccessRate, 0.99) + assert.equal(result.thresholds.every(({ pass }) => pass), true) + assert.equal( + result.thresholds.find(({ name }) => name === 'recovery_duration_ms')?.limit, + 840_000 + ) +}) + +for (const [name, mutate, failedThreshold] of [ + [ + 'target connection ceiling', + (report) => { + report.load.targetCells[0].peakConnections = 601 + }, + 'peak_target_connections' + ], + [ + 'migration expiration', + (report) => { + report.outcomes.migrationExpirations = 1 + }, + 'migration_expirations' + ], + [ + 'migration abort', + (report) => { + report.outcomes.migrationAborts = 1 + }, + 'migration_aborts' + ], + [ + 'full-wave registration deadline', + (report) => { + report.outcomes.targetRegistrationDurationMs = 300_001 + }, + 'target_registration_duration_ms' + ], + [ + 'legacy assignment background shape', + (report) => { + report.load.backgroundAssignment503PerMinute = 100 + }, + 'background_assignment_503_per_minute' + ], + [ + 'production director instance topology', + (report) => { + report.outcomes.directorPeakInstances = 1 + }, + 'director_peak_instances' + ], + [ + 'old/new rollout overlap topology', + (report) => { + report.outcomes.rolloutOverlapPeakInstances = 2 + }, + 'rollout_overlap_peak_instances' + ], + [ + 'old revision shared admission mode', + (report) => { + report.environment.rolloutOldResolvePrioritySlots = 1 + }, + 'rollout_old_resolve_priority_slots' + ], + [ + 'old/new rollout overlap resolve availability', + (report) => { + report.outcomes.rolloutOverlapResolveOverload = 2 + }, + 'rollout_overlap_resolve_overload_rate' + ], + [ + 'resolve availability', + (report) => { + report.outcomes.resolve2xx = 940 + report.outcomes.resolveOverload = 20 + }, + 'resolve_success_rate' + ], + [ + 'pool wait', + (report) => { + report.outcomes.poolWaitP95Ms = 500 + }, + 'pool_wait_p95_ms' + ], + [ + 'database CPU', + (report) => { + report.outcomes.databaseCpuPercentMax = 85 + }, + 'database_cpu_percent_max' + ], + [ + 'recovery deadline', + (report) => { + report.outcomes.recoveryDurationMs = 840_001 + }, + 'recovery_duration_ms' + ], + [ + 'non-public database maintenance', + (report) => { + report.outcomes.maintenanceFailures = 1 + }, + 'maintenance_failures' + ] +]) { + test(`fails closed on ${name}`, () => { + const report = passingReport() + mutate(report) + const result = evaluateRecoveryWaveReport(report) + + assert.equal(result.status, 'FAIL') + assert.equal( + result.thresholds.find(({ name: thresholdName }) => thresholdName === failedThreshold)?.pass, + false + ) + }) +} + +test('rejects production provenance and unexpected report fields', () => { + const productionProject = passingReport() + productionProject.environment.projectId = 'onorca-cloud' + assert.throws( + () => evaluateRecoveryWaveReport(productionProject), + /isolated non-production project/ + ) + + const productionOrigin = passingReport() + productionOrigin.environment.directorOrigin = 'https://relay.onorca.dev' + assert.throws( + () => evaluateRecoveryWaveReport(productionOrigin), + /isolated non-production origin/ + ) + + const extraField = passingReport() + extraField.environment.accessToken = 'must-not-be-accepted' + assert.throws(() => evaluateRecoveryWaveReport(extraField), /unexpected or missing fields/) +}) + +function passingReport() { + return { + schemaVersion: 1, + environment: { + projectId: 'onorca-cloud-staging', + directorOrigin: 'https://relay-staging.onorca.dev', + databaseVcpu: 2, + databasePoolMax: 3, + publicConcurrentMax: 2, + resolvePrioritySlots: 1, + directorMinInstances: 1, + directorMaxInstances: 2, + cloudRunConcurrency: 80, + rolloutOldPublicConcurrentMax: 2, + rolloutOldResolvePrioritySlots: 0, + rolloutNewPublicConcurrentMax: 2, + rolloutNewResolvePrioritySlots: 1 + }, + load: { + drainingDesktops: 800, + backgroundRequestsPerMinute: 11_000, + backgroundAssignmentRequestsPerMinute: 10_950, + backgroundAssignment503PerMinute: 9_500, + targetConnectionCap: 600, + targetCells: [ + { cellId: 'target-a', peakConnections: 425, recoveredControls: 400 }, + { cellId: 'target-b', peakConnections: 419, recoveredControls: 400 } + ] + }, + outcomes: { + migrationExpirations: 0, + migrationAborts: 0, + transactionRetryExhaustions: 0, + keyProvenTargetRegistrations: 800, + oldestMigrationLeaseRemainingAtDrainMs: 660_000, + targetRegistrationDurationMs: 240_000, + assignmentSuccessesPerMinuteBaseline: 1_400, + assignmentSuccessesPerMinuteMinimum: 1_330, + eligibleResolveRequests: 1_000, + resolve2xx: 990, + resolveOverload: 5, + readinessChecks: 180, + readinessFailures: 0, + maintenanceOperations: 30, + maintenanceFailures: 0, + directorPeakInstances: 2, + rolloutOverlapPeakInstances: 4, + rolloutOverlapPeakPublicOperations: 8, + rolloutOverlapEligibleResolveRequests: 100, + rolloutOverlapResolve2xx: 99, + rolloutOverlapResolveOverload: 0, + rolloutOverlapReadinessFailures: 0, + rolloutOverlapPoolWaitP95Ms: 180, + rolloutOverlapDatabaseCpuPercentMax: 78, + poolWaitP95Ms: 120, + poolWaitMaxMs: 900, + databaseCpuPercentP95: 55, + databaseCpuPercentMax: 72, + recoveryDurationMs: 360_000 + } + } +} diff --git a/cloud/dev/scripts/relay-region-observation-evidence.mjs b/cloud/dev/scripts/relay-region-observation-evidence.mjs new file mode 100644 index 00000000000..1c23a9c9b00 --- /dev/null +++ b/cloud/dev/scripts/relay-region-observation-evidence.mjs @@ -0,0 +1,152 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +const WINDOW_MS = 24 * 60 * 60_000 +const BUCKET_MS = 60 * 60_000 +const METRICS = [ + 'requestedRegionsDelta', + 'selectedRegionsDelta', + 'regionFallbacksDelta', + 'unavailableRegionsDelta' +] +const REGION_KEYS = new Set(['asia-east2', 'us-central1', 'unhinted']) + +function integer(value, name) { + const parsed = Number(value) + if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`) + return parsed +} + +function digest(value, name) { + if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`) + return value +} + +function metric(value, name) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${name} is invalid`) + } + return Object.fromEntries(Object.entries(value).map(([key, count]) => { + if (!REGION_KEYS.has(key)) throw new Error(`${name} has an unknown aggregate key`) + return [key, integer(count, `${name}.${key}`)] + })) +} + +function sumMetric(total, value) { + for (const [key, count] of Object.entries(value)) total[key] = (total[key] ?? 0) + count +} + +function evidenceSha256(evidence) { + return createHash('sha256').update(JSON.stringify(evidence)).digest('hex') +} + +export function createRegionObservationEvidence(entries, bindings, now = Date.now()) { + if (!Array.isArray(entries)) throw new Error('runtime metrics response must be an array') + if (!/^[0-9a-f]{40}$/.test(bindings.commitSha ?? '')) throw new Error('commit SHA is invalid') + const directorImageDigest = digest(bindings.directorImageDigest, 'director digest') + const selectorGeneration = integer(bindings.selectorGeneration, 'selector generation') + const controlGeneration = integer(bindings.controlGeneration, 'control generation') + const start = now - WINDOW_MS + const buckets = Array.from({ length: 24 }, () => 0) + const totals = Object.fromEntries(METRICS.map((name) => [name, {}])) + let samples = 0 + for (const entry of entries) { + const timestamp = Date.parse(entry?.timestamp ?? '') + const payload = entry?.jsonPayload + if ( + !Number.isFinite(timestamp) || + timestamp < start || + timestamp > now + 60_000 || + payload?.event !== 'orca_relay_runtime_metrics' || + payload.role !== 'director' + ) continue + const bucket = Math.min(23, Math.floor((timestamp - start) / BUCKET_MS)) + buckets[bucket] += 1 + samples += 1 + for (const name of METRICS) sumMetric(totals[name], metric(payload[name], name)) + } + if (buckets.some((count) => count === 0)) { + throw new Error('24-hour region evidence has a missing hourly bucket') + } + if ( + !Number.isSafeInteger(totals.requestedRegionsDelta['asia-east2']) || + totals.requestedRegionsDelta['asia-east2'] < 1 || + !Number.isSafeInteger(totals.selectedRegionsDelta['asia-east2']) || + totals.selectedRegionsDelta['asia-east2'] < 1 + ) throw new Error('24-hour region evidence has no Asia request and selection activity') + const evidence = { + v: 1, + commitSha: bindings.commitSha, + directorImageDigest, + selectorGeneration, + controlGeneration, + windowStartedAt: start, + windowEndedAt: now, + hourlySampleCounts: buckets, + samples, + totals + } + return { evidence, sha256: evidenceSha256(evidence) } +} + +export function verifyRegionObservationEvidence(sealed, bindings) { + if ( + sealed?.sha256 !== evidenceSha256(sealed?.evidence) || + sealed.evidence?.commitSha !== bindings.commitSha || + sealed.evidence?.directorImageDigest !== bindings.directorImageDigest || + sealed.evidence?.selectorGeneration !== Number(bindings.selectorGeneration) || + sealed.evidence?.controlGeneration !== Number(bindings.controlGeneration) || + !Array.isArray(sealed.evidence?.hourlySampleCounts) || + sealed.evidence.hourlySampleCounts.length !== 24 || + sealed.evidence.hourlySampleCounts.some((count) => integer(count, 'bucket') < 1) || + integer(sealed.evidence?.totals?.requestedRegionsDelta?.['asia-east2'], 'Asia requests') < 1 || + integer(sealed.evidence?.totals?.selectedRegionsDelta?.['asia-east2'], 'Asia selections') < 1 + ) throw new Error('sealed 24-hour region evidence does not match enable authority') + return sealed.evidence +} + +function values(argv) { + const result = {} + for (let index = 0; index < argv.length; index += 2) { + if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) { + throw new Error('invalid arguments') + } + result[argv[index].slice(2)] = argv[index + 1] + } + return result +} + +async function stdinJson(input) { + const chunks = [] + for await (const chunk of input) chunks.push(chunk) + return JSON.parse(Buffer.concat(chunks).toString('utf8')) +} + +export async function main(argv = process.argv.slice(2), input = process.stdin) { + const command = argv.shift() + const args = values(argv) + const bindings = { + commitSha: args['commit-sha'], + directorImageDigest: args['director-image-digest'], + selectorGeneration: args['selector-generation'], + controlGeneration: args['control-generation'] + } + if (command === 'create') { + const sealed = createRegionObservationEvidence(await stdinJson(input), bindings) + process.stdout.write(`${JSON.stringify(sealed)}\n`) + return + } + if (command === 'verify') { + verifyRegionObservationEvidence(JSON.parse(readFileSync(args.file, 'utf8')), bindings) + return + } + throw new Error('unknown region observation evidence command') +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/relay-region-observation-evidence.test.mjs b/cloud/dev/scripts/relay-region-observation-evidence.test.mjs new file mode 100644 index 00000000000..3426c5da5d6 --- /dev/null +++ b/cloud/dev/scripts/relay-region-observation-evidence.test.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + createRegionObservationEvidence, + verifyRegionObservationEvidence +} from './relay-region-observation-evidence.mjs' + +const now = Date.parse('2026-08-14T12:00:00Z') +const bindings = { + commitSha: 'a'.repeat(40), + directorImageDigest: `sha256:${'b'.repeat(64)}`, + selectorGeneration: 11, + controlGeneration: 4 +} + +function entries() { + return Array.from({ length: 24 }, (_, index) => ({ + timestamp: new Date(now - (index * 60 + 30) * 60_000).toISOString(), + jsonPayload: { + event: 'orca_relay_runtime_metrics', + role: 'director', + requestedRegionsDelta: { 'asia-east2': index === 0 ? 2 : 0 }, + selectedRegionsDelta: { 'asia-east2': index === 0 ? 1 : 0 }, + regionFallbacksDelta: { 'asia-east2': index === 0 ? 1 : 0 }, + unavailableRegionsDelta: {} + } + })) +} + +test('seals all 24 hourly aggregate region buckets', () => { + const sealed = createRegionObservationEvidence(entries(), bindings, now) + assert.equal(sealed.evidence.hourlySampleCounts.length, 24) + assert.equal(sealed.evidence.totals.requestedRegionsDelta['asia-east2'], 2) + assert.equal(verifyRegionObservationEvidence(sealed, bindings).samples, 24) +}) + +test('rejects missing coverage, missing Asia activity, and changed bindings', () => { + assert.throws(() => createRegionObservationEvidence(entries().slice(1), bindings, now), /missing hourly/) + const noAsia = entries().map((entry) => ({ + ...entry, + jsonPayload: { + ...entry.jsonPayload, + requestedRegionsDelta: {}, + selectedRegionsDelta: {} + } + })) + assert.throws(() => createRegionObservationEvidence(noAsia, bindings, now), /no Asia/) + const sealed = createRegionObservationEvidence(entries(), bindings, now) + assert.throws(() => verifyRegionObservationEvidence(sealed, { + ...bindings, + commitSha: 'c'.repeat(40) + }), /does not match/) +}) diff --git a/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs b/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs new file mode 100644 index 00000000000..5eab757255a --- /dev/null +++ b/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs @@ -0,0 +1,193 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { relayWorkflowUrl } from './relay-repository.mjs' + +function workflow(name) { + return readFileSync( + fileURLToPath(relayWorkflowUrl(name)), + 'utf8' + ) +} + +test('same-cap wrapper is reusable, canary-bound, and sequential', () => { + const wrapper = workflow('deploy-relay-production-same-cap.yml') + const job = workflow('deploy-relay-production-same-cap-job.yml') + assert.match(wrapper, /options: \[verify, canary-apply, batch-apply, rollback\]/) + assert.match(wrapper, /relay-same-cap-canary-\$\{\{ inputs\.canary-run-id \}\}/) + assert.match(wrapper, /needs: \[gate, cell_1\]/) + assert.match(wrapper, /needs: \[gate, cell_2\]/) + assert.match(wrapper, /needs: \[gate, cell_3\]/) + assert.match(job, /on:\n workflow_call:/) + assert.match(job, /c27\|c28\|c29/) + assert.match(job, /EXPECTED_HARD_CAP=3000/) + assert.match(job, /EXPECTED_REGION=asia-east2/) + assert.match(job, /--hard-cap "\$\{EXPECTED_HARD_CAP\}"/) + assert.match(job, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/) + assert.match(job, /--argjson protocol "\$\{PREDECESSOR_REHOME_PROTOCOL\}"/) + assert.match(job, /runtime predecessor mismatch fields=/) + // A rollback interrupted between apply and restore must be resumable. + assert.match(job, /ROLLBACK_RESUME=true/) + assert.match(job, /test "\$\{LIVE_IMAGE_DIGEST\}" = "\$\{DESIRED_IMAGE_DIGEST\}"/) + // Resume must skip BOTH the drain (no restart will clear the flag) and the + // apply (state already converged), and prove convergence instead. + assert.match( + job, + /Reversibly isolate and drain only the selected cell\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/ + ) + assert.match( + job, + /Apply only the selected same-cap template and MIG\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/ + ) + assert.match( + job, + /Require converged Terraform state and a stable MIG on resume\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME == 'true' \}\}/ + ) + assert.match(job, /resume found unconverged resources/) + // A canary or batch cell that failed before its template apply also + // resumes here with template drift from repo changes since its last roll; + // only a plan the reviewed validator approves for the image the cell + // already serves may pass, and resume still applies nothing. + assert.match(job, /requiring reviewed rollback-image drift/) + assert.match( + job, + /--image "\$\{DESIRED_IMAGE\}" \\\n {16}--rollback-image "\$\{DESIRED_IMAGE\}"/ + ) + // The relaxation is only safe if the reviewed validator actually runs on + // the NON-converged branch, in same-cap-cell mode, with the trust config + // the validator requires, restricted to the template-and-MIG change pair. + assert.match( + job, + /if ! terraform -chdir=infra\/terraform show -json[\s\S]{0,220}\| length == 0' >\/dev\/null\n then\n/ + ) + assert.match( + job, + /requiring reviewed rollback-image drift'\n[\s\S]{0,400}?\n {16}--mode same-cap-cell --cell-id "\$\{TARGET_CELL_ID\}" \\\n/ + ) + assert.match( + job, + /Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/ + ) + assert.match( + job, + /--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/ + ) + assert.match(job, /host-drain \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/) + assert.match(job, /resume requires the isolated migration-only cell/) + assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/) + assert.match(job, /\(.regionalRehomeProtocol \/\/ 0\) == \$protocol/) + assert.match(job, /\(\.draining == false or \$drainingOk\)/) + // Selector expectations must follow the mutations' returned generations, + // not fixed offsets: isolate is a no-op on a cell a failed canary already + // isolated, and the restore inspect must expect post-restore membership. + assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/) + assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/) + assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/) + assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ACTIVATE\}"/) + assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/) + assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/) + assert.match(job, /FAILSAFE_GENERATION/) + // Later batch waves start after ~16-min predecessor rolls, so BOTH evidence + // age checks must scale by wave or cell_2+ can never pass; the bound's + // per-wave step is the cell job timeout, so the two must move together. + assert.match(job, /--required-migration-policy strict \\\n --wave-index "\$\{WAVE_INDEX\}"/) + assert.match(job, /dry-run\.state\.json" \\\n --wave-index "\$\{WAVE_INDEX\}" "\$\{RETRY_ARGS\[@\]\}"/) + assert.match(job, /timeout-minutes: 75/) + // Both age gates step by the cell job timeout above; the constant is + // duplicated across the two languages, so pin each copy to it. + for (const source of [ + '../../dev/scripts/relay-monitor-evidence.mjs', + '../../apps/relay-ops/src/incident-live-preflight-cli.ts' + ]) { + const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8') + assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/) + assert.match(body, /\^\[0-3\]\$/) + } + // Aged-evidence replay via job re-runs is fenced: mutations are + // single-dispatch, so a failed cell needs a fresh gate and monitor run. + assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/) + for (const index of [0, 1, 2, 3]) { + assert.match(wrapper, new RegExp(`wave-index: '${index}'`)) + } + assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/) + assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 2\)/) + assert.match(job, /\$region == "us-central1" and \$protocol == 0 and [.]region == null/) + assert.match(job, /[.]regionalRehomeProtocol \/\/ 0/) + assert.match(job, /runtime predecessor normalized legacy fields=/) + assert.match(job, /probe-relay-rehome-trust[.]mjs/) + assert.doesNotMatch(job, /service_account: \$\{\{ vars\.PRODUCTION_GCP_RELAY_(?:DIRECTOR_)?RUNTIME_SERVICE_ACCOUNT/) + assert.doesNotMatch(job, /roles\/iam\.serviceAccountTokenCreator/) +}) + +// Why: the same-cap caller defines release_lease itself, and a caller-defined job presents the +// caller as job_workflow_ref, so the pair must admit the caller alongside its reusable job. +test('shared deploy WIF admits the exact same-cap reusable workflow pair and the caller itself', () => { + const terraform = readFileSync( + fileURLToPath(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url)), + 'utf8' + ) + const providerStart = terraform.indexOf( + 'resource "google_iam_workload_identity_pool_provider" "github"' + ) + const providerEnd = terraform.indexOf('\nresource "', providerStart + 1) + const sharedProvider = terraform.slice(providerStart, providerEnd) + assert.ok(providerStart >= 0 && providerEnd > providerStart) + assert.match(sharedProvider, /local\.relay_github_workflow_conditions\["github"\]/) + // The pairing itself now lives in the clause the provider renders, once per accepted repository. + assert.match( + terraform, + /assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main' && \(assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_job_workflow_file\}@refs\/heads\/main' \|\| assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main'\)/ + ) +}) + +test('pause and disable precede optional installation and cloud diagnostics', () => { + const job = workflow('operate-relay-production-rehome-job.yml') + const emergency = job.indexOf('Apply emergency durable pause or disable before diagnostics') + const install = job.indexOf('pnpm install --frozen-lockfile') + const revision = job.indexOf('Verify exact serving and rollback director identities') + assert.ok(emergency > 0) + assert.ok(emergency < install) + assert.ok(emergency < revision) + assert.match(job, /inputs\.mode == 'pause' \|\| inputs\.mode == 'disable'/) + assert.match(job, /Seal 24-hour aggregate region observation evidence/) + assert.match(job, /--freshness=25h --limit=30000/) + assert.match(job, /relay-region-observation-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/) + assert.match(job, /test "\$\{RATE_PER_MINUTE\}" = 10/) +}) + +test('a failed enable independently restores and verifies durable disabled state', () => { + const job = workflow('operate-relay-production-rehome-job.yml') + const enable = job.indexOf('Apply exact durable regional rehome enable') + const evidence = job.indexOf('Read fresh aggregate completion and abort evidence') + const summary = job.indexOf('Publish aggregate control evidence') + const recovery = job.indexOf('Fail closed after an unsuccessful enable run') + assert.ok(enable > 0 && enable < evidence && evidence < summary && summary < recovery) + const recoveryStep = job.slice(recovery) + assert.match( + recoveryStep, + /failure\(\) && inputs\.mode == 'enable' && steps\.google-auth\.outcome == 'success'/ + ) + assert.match(recoveryStep, /--mode recover-enable/) + assert.match(recoveryStep, /--expected-control-generation "\$\{EXPECTED_CONTROL_GENERATION\}"/) + assert.match(recoveryStep, /RECOVER_FAILED_REGIONAL_REHOME_ENABLE/) + assert.match(recoveryStep, /\.control\.enabled == false/) + assert.doesNotMatch(recoveryStep, /gcloud|pnpm/) +}) + +test('director rollout has a strict one-time identity bootstrap', () => { + const workflowBody = workflow('deploy-relay-production-director.yml') + const script = readFileSync( + fileURLToPath(new URL('./deploy-relay-blue-green.mjs', import.meta.url)), + 'utf8' + ) + assert.match(workflowBody, /BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY/) + assert.match(workflowBody, /--predecessor-runtime-service-account/) + assert.match(workflowBody, /--expected-rehome-generation/) + assert.match(script, /args\.push\('--service-account', config\['runtime-service-account'\]\)/) + assert.match(script, /director predecessor runtime service account does not match/) + const candidateProof = script.indexOf('await verifyRehomeDisabled(candidate.origin)') + const trafficMove = script.indexOf('operations.updateTraffic(config, [`--to-tags=') + assert.ok(candidateProof > 0 && candidateProof < trafficMove) + assert.equal(script.indexOf('verifyRehomeDisabled', trafficMove), -1) +}) diff --git a/cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs b/cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs new file mode 100644 index 00000000000..82bc2fb522a --- /dev/null +++ b/cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs @@ -0,0 +1,65 @@ +import { pathToFileURL } from 'node:url' + +const INVENTORY = /^\[orca-relay\] regional rehome inventory active=(\d+) awaitingReceipt=(\d+) targetRegistered=(\d+) completedLast24Hours=(\d+) abortedLast24Hours=(\d+) oldestActiveAgeMs=(none|\d+)$/ + +function count(value, name) { + const parsed = Number(value) + if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`) + return parsed +} + +export function parseRegionalRehomeInventory(entries, options = {}) { + if (!Array.isArray(entries)) throw new Error('logging response must be an array') + const parsed = entries.flatMap((entry) => { + const match = INVENTORY.exec(entry?.textPayload ?? '') + const timestamp = Date.parse(entry?.timestamp ?? '') + if (!match || !Number.isFinite(timestamp)) return [] + return [{ + timestamp, + active: count(match[1], 'active'), + awaitingReceipt: count(match[2], 'awaiting receipt'), + targetRegistered: count(match[3], 'target registered'), + completedLast24Hours: count(match[4], 'completed'), + abortedLast24Hours: count(match[5], 'aborted'), + oldestActiveAgeMs: match[6] === 'none' ? null : count(match[6], 'oldest active age') + }] + }).sort((left, right) => right.timestamp - left.timestamp) + if (parsed.length === 0) throw new Error('no aggregate regional rehome inventory evidence') + const latest = parsed[0] + const now = options.now ?? Date.now() + const maxAgeMs = options.maxAgeMs ?? 15 * 60_000 + if (latest.timestamp > now + 60_000 || latest.timestamp < now - maxAgeMs) { + throw new Error('aggregate regional rehome inventory evidence is stale') + } + return latest +} + +function argumentsMap(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) { + throw new Error('invalid arguments') + } + values[argv[index].slice(2)] = argv[index + 1] + } + return values +} + +export async function main(argv = process.argv.slice(2), input = process.stdin) { + const values = argumentsMap(argv) + const maxAgeMs = count(values['max-age-ms'] ?? 900_000, '--max-age-ms') + const chunks = [] + for await (const chunk of input) chunks.push(chunk) + const evidence = parseRegionalRehomeInventory( + JSON.parse(Buffer.concat(chunks).toString('utf8')), + { maxAgeMs } + ) + process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_aggregate_evidence', ...evidence })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs b/cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs new file mode 100644 index 00000000000..2ce2627fb93 --- /dev/null +++ b/cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { parseRegionalRehomeInventory } from './relay-rehome-aggregate-evidence.mjs' + +const now = Date.parse('2026-08-14T12:00:00Z') + +test('selects the newest fresh aggregate-only regional rehome inventory', () => { + const result = parseRegionalRehomeInventory([ + { + timestamp: '2026-08-14T11:58:00Z', + textPayload: '[orca-relay] regional rehome inventory active=2 awaitingReceipt=1 targetRegistered=1 completedLast24Hours=9 abortedLast24Hours=0 oldestActiveAgeMs=30000' + }, + { + timestamp: '2026-08-14T11:50:00Z', + textPayload: '[orca-relay] regional rehome inventory active=1 awaitingReceipt=0 targetRegistered=1 completedLast24Hours=8 abortedLast24Hours=0 oldestActiveAgeMs=none' + } + ], { now, maxAgeMs: 5 * 60_000 }) + assert.deepEqual(result, { + timestamp: Date.parse('2026-08-14T11:58:00Z'), + active: 2, + awaitingReceipt: 1, + targetRegistered: 1, + completedLast24Hours: 9, + abortedLast24Hours: 0, + oldestActiveAgeMs: 30_000 + }) +}) + +test('rejects stale, malformed, and identity-bearing lookalikes', () => { + assert.throws(() => parseRegionalRehomeInventory([{ + timestamp: '2026-08-14T11:00:00Z', + textPayload: '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0 completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none' + }], { now, maxAgeMs: 5 * 60_000 }), /stale/) + assert.throws(() => parseRegionalRehomeInventory([{ + timestamp: '2026-08-14T11:59:00Z', + textPayload: '[orca-relay] regional rehome inventory active=0 hostId=secret' + }], { now }), /no aggregate/) +}) diff --git a/cloud/dev/scripts/relay-repository.mjs b/cloud/dev/scripts/relay-repository.mjs new file mode 100644 index 00000000000..bf41bed8012 --- /dev/null +++ b/cloud/dev/scripts/relay-repository.mjs @@ -0,0 +1,29 @@ +import { readFileSync } from 'node:fs' + +// Single place naming the repository the Relay workflows live in and where their files sit. The +// public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the +// owning repository, so only this module changes: nothing else may restate any of the three. +export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca' + +export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-' + +// Where .github/workflows sits relative to this file. Workflows stay at the repository root while +// this tree moves under cloud/, so the depth changes at the copy even though the layout does not. +export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url) + +export function relayWorkflowFile(name) { + return `${RELAY_WORKFLOW_FILE_PREFIX}${name}` +} + +// Repository-relative path, the shape GitHub reports in workflow_ref and evidence payloads. +export function relayWorkflowPath(name) { + return `.github/workflows/${relayWorkflowFile(name)}` +} + +export function relayWorkflowUrl(name) { + return new URL(relayWorkflowFile(name), RELAY_WORKFLOW_DIRECTORY) +} + +export function readRelayWorkflow(name) { + return readFileSync(relayWorkflowUrl(name), 'utf8') +} diff --git a/cloud/dev/scripts/relay-repository.test.mjs b/cloud/dev/scripts/relay-repository.test.mjs new file mode 100644 index 00000000000..56383db4a1e --- /dev/null +++ b/cloud/dev/scripts/relay-repository.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict' +import { readdirSync, readFileSync } from 'node:fs' +import test from 'node:test' +import { fileURLToPath } from 'node:url' +import { + RELAY_GITHUB_REPOSITORY, + RELAY_WORKFLOW_FILE_PREFIX, + readRelayWorkflow, + relayWorkflowFile, + relayWorkflowPath, + relayWorkflowUrl +} from './relay-repository.mjs' + +const directory = fileURLToPath(new URL('.', import.meta.url)) +// The Relay copy takes the scripts named for it. Everything else stays with the applications. +const relayScripts = readdirSync(directory) + .filter((name) => name.includes('relay') && name.endsWith('.mjs')) + .filter((name) => !name.startsWith('relay-repository.')) + +test('workflow identity is derived, never restated', () => { + assert.equal(relayWorkflowFile('deploy-relay-staging.yml'), `${RELAY_WORKFLOW_FILE_PREFIX}deploy-relay-staging.yml`) + assert.equal(relayWorkflowPath('deploy-relay-staging.yml'), `.github/workflows/${relayWorkflowFile('deploy-relay-staging.yml')}`) + assert.ok(relayWorkflowUrl('deploy-relay-staging.yml').pathname.endsWith(relayWorkflowPath('deploy-relay-staging.yml'))) + assert.match(readRelayWorkflow('deploy-relay-staging.yml'), /^name:/m) + assert.match(RELAY_GITHUB_REPOSITORY, /^[\w.-]+\/[\w.-]+$/) +}) + +// Why: the public-repo copy changes the owning repository, the workflow filenames, and the depth +// this tree sits at. Each has to be one edit here, so no Relay script may restate any of them. +test('no Relay script restates the repository or the workflow directory', () => { + for (const name of relayScripts) { + const text = readFileSync(`${directory}${name}`, 'utf8') + assert.doesNotMatch(text, /stablyai\//, `${name} restates the GitHub repository`) + assert.doesNotMatch(text, /\.github\/workflows/, `${name} restates the workflow directory`) + } +}) diff --git a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs new file mode 100644 index 00000000000..1f0e6fce3ad --- /dev/null +++ b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs @@ -0,0 +1,159 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { relayWorkflowFile, relayWorkflowUrl } from './relay-repository.mjs' + +const workflow = readFileSync( + relayWorkflowUrl('prove-relay-staging-capacity.yml'), + 'utf8' +) +const recoveryWorkflow = readFileSync( + relayWorkflowUrl('recover-relay-staging-c4-image.yml'), + 'utf8' +) +const requeueWorkflow = readFileSync( + relayWorkflowUrl('requeue-relay-staging-c4-recovery.yml'), + 'utf8' +) +const githubActions = readFileSync( + new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url), + 'utf8' +) +const cells = readFileSync( + new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), + 'utf8' +) +const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8') +const stagingTfvars = readFileSync( + new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url), + 'utf8' +) +const productionTfvars = readFileSync( + new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), + 'utf8' +) + +const launchDigest = '5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563' + +// Scoped to the Asia cells by name: the production capacity cells now serve this digest too, +// so a file-wide count no longer isolates Asia. +const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'] + +function productionCell(cellId) { + const start = productionTfvars.indexOf(`"${cellId}"`) + assert.notEqual(start, -1, `${cellId} is missing`) + return productionTfvars.slice(start, productionTfvars.indexOf('\n }', start)) +} + +test('pins staging C4 and all production Asia cells to the same launch image', () => { + assert.equal(stagingTfvars.match(new RegExp(launchDigest, 'g'))?.length, 1) + for (const cellId of asiaCells) { + assert.match(productionCell(cellId), new RegExp(`relay@sha256:${launchDigest}"`), cellId) + } + assert.match(recoveryWorkflow, new RegExp(`TARGET_IMAGE_DIGEST: sha256:${launchDigest}`)) +}) + +test('refreshes only empty staging C4 through the trusted capacity identity', () => { + const refresh = workflow.slice(workflow.indexOf(' refresh-asia-c4-image:')) + assert.match(refresh, /terraform_version: 1\.15\.8/) + assert.doesNotMatch(refresh, /terraform_version: 1\.5\.7/) + assert.match(refresh, /github\.ref == 'refs\/heads\/main'/) + assert.match(refresh, /REFRESH_STAGING_ASIA_C4_IMAGE/) + assert.match(refresh, /APPROVED_PREDECESSOR_IMAGE_DIGEST/) + assert.match(refresh, /--activity quiescent/) + assert.match(refresh, /--admission migration-only/) + assert.match(refresh, /fence_digests="\$\{PREDECESSOR_IMAGE_DIGEST\}"/) + assert.match(refresh, /--expected-image-digests "\$\{TARGET_IMAGE_DIGEST\}"/) + assert.match(refresh, /--mode same-cap-image/) + assert.match(refresh, /test "\$\(jq -r '\.changes'/) + assert.match(refresh, /REFRESH_PHASE=\$\{refresh_phase\}/) + assert.match(refresh, /MUTATION_STARTED=false/) + assert.match(refresh, /MIG_STABLE_AT_MS=/) + assert.match(refresh, /PLAN_CHANGES=/) + assert.match(refresh, /obsolete-template-delete/) + assert.match( + refresh, + /\*:manager-convergence\|\*:replacement-with-obsolete-template\) refresh_phase=converging/ + ) + assert.match(refresh, /--mode isolate/) + assert.match(refresh, /--mode verify/) + assert.match(refresh, /\.status\.runtime\.ready/) + assert.match(refresh, /\.status\.runtime\.startedAt/) + assert.match(refresh, /\.status\.runtime\.lastHeartbeatAt/) + assert.match(refresh, /--runtime unavailable/) + const plan = refresh.indexOf('Save, validate, and classify the exact C4 plan') + const currentState = refresh.indexOf('Verify the exact selector and current C4 state') + const isolate = refresh.indexOf('--mode isolate') + const apply = refresh.indexOf('terraform -chdir=infra/terraform apply -auto-approve') + assert.ok(plan < currentState && currentState < isolate && isolate < apply) + assert.match(refresh, /Require an empty targeted Terraform readback/) +}) + +test('recovers a failed or cancelled C4 refresh from an independent workflow', () => { + assert.match(recoveryWorkflow, /terraform_version: 1\.15\.8/) + assert.doesNotMatch(recoveryWorkflow, /terraform_version: 1\.5\.7/) + assert.match(recoveryWorkflow, /workflow_run:/) + assert.match(recoveryWorkflow, /workflows: \[Prove Relay Staging Capacity\]/) + assert.match(recoveryWorkflow, /RECOVER_STAGING_ASIA_C4_IMAGE/) + assert.match(recoveryWorkflow, /outputs:\n\s+recover: \$\{\{ steps\.trigger\.outputs\.recover \}\}/) + assert.match(recoveryWorkflow, /if test "\$\{count\}" = 0; then\n\s+echo "recover=false"/) + assert.match(recoveryWorkflow, /needs: gate\n\s+if: \$\{\{ needs\.gate\.outputs\.recover == 'true' \}\}/) + assert.match(recoveryWorkflow, /concurrency:\n\s+group: relay-staging-mutation/) + assert.match(recoveryWorkflow, /group: relay-staging-mutation/) + assert.match(recoveryWorkflow, /\.name == "refresh-asia-c4-image"/) + assert.match(recoveryWorkflow, /PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13/) + assert.match(recoveryWorkflow, /TARGET_IMAGE_DIGEST: sha256:5aedbca5/) + assert.match(recoveryWorkflow, /id: preflight-auth/) + assert.match(recoveryWorkflow, /id: verify-auth/) + assert.match(recoveryWorkflow, /\.status\.runtime\.ready/) + assert.match(recoveryWorkflow, /--runtime unavailable/) + assert.match(recoveryWorkflow, /current_digest.*\^sha256:\[a-f0-9\]\{64\}\$/) + assert.match(recoveryWorkflow, /expected_digests="\$\{expected_digests\},\$\{current_digest\}"/) + assert.match(recoveryWorkflow, /--timeout-ms 240000/) + assert.doesNotMatch(recoveryWorkflow, /--timeout-ms 900000/) + assert.match(recoveryWorkflow, /-var manage_artifact_dns=false -lock-timeout=5m/) + assert.match(recoveryWorkflow, /--mode same-cap-image/) + assert.match(recoveryWorkflow, /test "\$\(jq -r '\.changes'/) + assert.equal(recoveryWorkflow.match(/\*:replacement-with-obsolete-template/g)?.length, 2) + assert.match( + recoveryWorkflow, + /\^\(replacement\|replacement-with-obsolete-template\|manager-convergence\)\$/ + ) + const preflightAuth = recoveryWorkflow.indexOf('id: preflight-auth') + const preflight = recoveryWorkflow.indexOf('Inspect the exact C4 recovery state') + const recoveryPlan = recoveryWorkflow.indexOf('Classify both exact recovery end states') + const apply = recoveryWorkflow.indexOf('Apply and stabilize the saved predecessor plan') + const restore = recoveryWorkflow.indexOf('Restart only when the plan did not replace C4') + const verifyAuth = recoveryWorkflow.indexOf('id: verify-auth') + const verify = recoveryWorkflow.indexOf('Verify the recovered image and unchanged isolation') + assert.ok(preflightAuth < preflight && preflight < recoveryPlan && recoveryPlan < apply) + assert.ok(apply < restore) + assert.ok(restore < verifyAuth && verifyAuth < verify) + assert.match(githubActions, /"recover-relay-staging-c4-image\.yml"/) +}) + +test('requeues a protected C4 recovery cancelled while pending', () => { + assert.match(requeueWorkflow, /workflows: \[Recover Relay Staging C4 Image\]/) + assert.match(requeueWorkflow, /conclusion == 'cancelled'/) + assert.match(requeueWorkflow, /permissions:\n\s+actions: write\n\s+contents: read/) + assert.match(requeueWorkflow, /group: relay-staging-c4-recovery-requeue/) + assert.match(requeueWorkflow, /\.name == "recover" and\n\s+\.conclusion == "cancelled"/) + assert.match(requeueWorkflow, /\.started_at == null/) + assert.match(requeueWorkflow, /\.name == "gate" and \.conclusion == "success"/) + assert.match(requeueWorkflow, /\.name == "recover" and \.status != "completed"/) + assert.match(requeueWorkflow, /actions\/runs\/\$\{run_id\}\/jobs\?filter=latest/) + assert.match(requeueWorkflow, /if test "\$\{active\}" != 0; then exit 0; fi/) + assert.ok(requeueWorkflow.includes(`gh workflow run ${relayWorkflowFile('recover-relay-staging-c4-image.yml')}`)) + assert.match(requeueWorkflow, /-f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE/) + assert.doesNotMatch(requeueWorkflow, /id-token: write/) + assert.doesNotMatch(requeueWorkflow, /relay-staging-mutation/) +}) + +test('keeps cell-only plans independent from service-account description drift', () => { + assert.match(cells, /runtime_service_account\s+= local\.relay_runtime_service_account_email/) + assert.match( + cells, + /rehome_director_service_account\s+= local\.relay_director_runtime_service_account_email/ + ) + assert.match(relay, /var\.environment == "staging" \? "Orca Relay"/) +}) diff --git a/cloud/dev/scripts/relay-staging-capacity-identity.test.mjs b/cloud/dev/scripts/relay-staging-capacity-identity.test.mjs new file mode 100644 index 00000000000..31a38d08124 --- /dev/null +++ b/cloud/dev/scripts/relay-staging-capacity-identity.test.mjs @@ -0,0 +1,269 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { relayWorkflowUrl } from './relay-repository.mjs' + +const terraform = readFileSync( + new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url), + 'utf8' +) +const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8') +const workflow = readFileSync( + relayWorkflowUrl('prove-relay-staging-capacity.yml'), + 'utf8' +) +const deployWorkflow = readFileSync( + relayWorkflowUrl('deploy-relay-staging.yml'), + 'utf8' +) +const publishWorkflow = readFileSync( + relayWorkflowUrl('publish-relay-production.yml'), + 'utf8' +) +const bootstrapWorkflow = readFileSync( + relayWorkflowUrl('bootstrap-relay-staging-capacity.yml'), + 'utf8' +) + +function resource(type, name) { + const start = terraform.indexOf(`resource "${type}" "${name}"`) + assert.notEqual(start, -1, `${type}.${name} is missing`) + const next = terraform.indexOf('\nresource "', start + 1) + return terraform.slice(start, next === -1 ? undefined : next) +} + +function terraformStringList(block, attribute) { + const match = block.match(new RegExp(`${attribute}\\s*=\\s*\\[([\\s\\S]*?)\\]`)) + assert.ok(match, `${attribute} is missing`) + return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1]) +} + +test('capacity workflow uses only its exact staging identity', () => { + assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) + assert.doesNotMatch(workflow, /vars\.STAGING_GCP_WORKLOAD_IDENTITY_PROVIDER/) + assert.doesNotMatch(workflow, /vars\.STAGING_GCP_DEPLOY_SERVICE_ACCOUNT/) + + const provider = resource( + 'google_iam_workload_identity_pool_provider', + 'github_staging_relay_capacity' + ) + // The three repository claims are pinned once in relay-shared.tf; every provider concatenates + // that list rather than restating the repository on its own. + assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/) + for (const boundary of [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'staging'" + ]) { + assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'))) + } + assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_capacity"\]/) + assert.deepEqual( + terraformStringList(terraform, 'github_staging_relay_capacity_workflow_files'), + [ + 'bootstrap-relay-staging-capacity.yml', + 'prove-relay-staging-capacity.yml', + 'recover-relay-staging-c4-image.yml' + ] + ) + assert.match( + terraform, + /for workflow_file in local\.github_staging_relay_capacity_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/ + ) + assert.doesNotMatch(terraform, /github_staging_relay_capacity_workflow_file\s*=/) +}) + +test('job gates do not read environment variables before the environment is attached', () => { + for (const source of [workflow, deployWorkflow, bootstrapWorkflow]) { + const jobGate = source.match(/^\s{4}if:.*$/m)?.[0] ?? '' + assert.doesNotMatch(jobGate, /STAGING_GCP_RELAY_CAPACITY_/) + } +}) + +test('capacity identity has bounded mutation and state permissions', () => { + const role = resource( + 'google_project_iam_custom_role', + 'github_staging_relay_capacity_mutation' + ) + assert.deepEqual(terraformStringList(role, 'permissions'), [ + 'compute.disks.create', + 'compute.healthChecks.use', + 'compute.images.useReadOnly', + 'compute.instanceGroupManagers.get', + 'compute.instanceGroupManagers.update', + 'compute.instances.create', + 'compute.instances.setLabels', + 'compute.instances.setMetadata', + 'compute.instances.setTags', + 'compute.instanceTemplates.create', + 'compute.instanceTemplates.delete', + 'compute.instanceTemplates.get', + 'compute.instanceTemplates.useReadOnly', + 'compute.networks.use', + 'compute.subnetworks.use', + 'compute.zoneOperations.get' + ]) + assert.doesNotMatch( + role, + /compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/ + ) + + for (const source of [workflow, bootstrapWorkflow]) { + assert.match(source, /instance-groups managed recreate-instances/) + assert.match(source, /--instances/) + assert.doesNotMatch(source, /rolling-action restart/) + } + + const state = resource( + 'google_storage_bucket_iam_member', + 'github_staging_relay_capacity_state' + ) + assert.match(state, /roles\/storage\.objectAdmin/) + assert.match(state, /objects\/terraform\/state\/default\.tfstate/) + assert.match(state, /objects\/terraform\/state\/default\.tflock/) + assert.doesNotMatch(state, /resource\.name\.startsWith/) + + const runtime = resource( + 'google_service_account_iam_member', + 'github_staging_relay_capacity_runtime_user' + ) + assert.match(runtime, /google_service_account\.relay_runtime\.name/) + assert.match(runtime, /roles\/iam\.serviceAccountUser/) + + const cloudRun = resource( + 'google_cloud_run_v2_service_iam_member', + 'github_staging_relay_capacity_developer' + ) + assert.match(cloudRun, /name\s*=\s*var\.relay_cloud_run_service_name/) + assert.doesNotMatch(cloudRun, /google_cloud_run_v2_service\.relay/) + + const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8') + const startup = readFileSync( + new URL('../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url), + 'utf8' + ) + assert.match(relay, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/) + assert.match(startup, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/) +}) + +test('capacity identity exposes only its provider and service account', () => { + assert.match(outputs, /output "github_staging_relay_capacity_workload_identity_provider"/) + assert.match(outputs, /output "github_staging_relay_capacity_service_account"/) +}) + +test('director capacity configuration stays on the audited blue-green path', () => { + assert.match(deployWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) + assert.match(deployWorkflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/) + assert.match(deployWorkflow, /expected-image-digest/) + assert.match(deployWorkflow, /var\.relay_gce_cells\["staging-gce-c4"\]\.image/) + assert.match(deployWorkflow, /init -reconfigure \\\n\s+-backend-config=backend\/staging\.hcl/) + assert.ok( + deployWorkflow.indexOf('id: google-auth') < + deployWorkflow.indexOf('Bind the request to the checked-in staging C4 image') + ) + assert.match(deployWorkflow, /artifacts docker images describe "\$\{IMAGE\}"/) + assert.doesNotMatch(deployWorkflow, /docker (?:build|push)/) + assert.match(workflow, /--director-cells-json "\$\{DESIRED_CELLS_JSON\}"/) + assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/) + assert.doesNotMatch(workflow, /--mode director/) +}) + +test('mirrors the exact production manifest through the production deploy identity', () => { + assert.match(publishWorkflow, /options: \[publish, mirror-staging\]/) + assert.match(publishWorkflow, /MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING/) + assert.match(publishWorkflow, /docker pull "\$\{source_image\}"/) + assert.match(publishWorkflow, /docker tag "\$\{source_image\}" "\$\{target_tag\}"/) + assert.match(publishWorkflow, /test "\$\{source_digest\}" = "\$\{MIRROR_DIGEST\}"/) + assert.match(publishWorkflow, /test "\$\{target_digest\}" = "\$\{MIRROR_DIGEST\}"/) + const mirrorWriter = resource( + 'google_artifact_registry_repository_iam_member', + 'github_production_relay_staging_mirror_writer' + ) + assert.match(mirrorWriter, /var\.environment == "staging"/) + assert.match(mirrorWriter, /roles\/artifactregistry\.writer/) + assert.match( + mirrorWriter, + /serviceAccount:orca-cloud-gha-deploy@onorca-cloud\.iam\.gserviceaccount\.com/ + ) +}) + +test('cells bootstrap one at a time with bounded deploy and capacity identities', () => { + assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT/) + assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/) + assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) + assert.match(bootstrapWorkflow, /--mode bootstrap-cell/) + assert.match(bootstrapWorkflow, /--cell-id "\$\{fallback_cell_id\}"/) + assert.match(bootstrapWorkflow, /google_compute_instance_template\.relay_gce_cell/) + assert.match(bootstrapWorkflow, /google_compute_instance_group_manager\.relay_gce_cell/) + assert.match(bootstrapWorkflow, /--mode restore-fallback/) + assert.doesNotMatch(bootstrapWorkflow, /target=google_cloud_run_v2_service\.relay/) + assert.ok( + bootstrapWorkflow.indexOf('id: deploy-auth') < bootstrapWorkflow.indexOf('id: capacity-auth') + ) + assert.match( + bootstrapWorkflow, + /restore_fallback\(\) \{[\s\S]*?verify_fallback[\s\S]*?--mode restore-fallback/ + ) + const rollCell = bootstrapWorkflow.slice(bootstrapWorkflow.indexOf('roll_cell()')) + assert.ok( + rollCell.indexOf('verify_fallback\n trap restore_fallback EXIT') < + rollCell.indexOf('--mode isolate') + ) + assert.match( + bootstrapWorkflow, + /staging-gce-c2 general[\s\S]*?trap restore_legacy_c3_fallback EXIT[\s\S]*?--mode isolate/ + ) + const normalize = bootstrapWorkflow.slice( + bootstrapWorkflow.indexOf('normalize_legacy_c3() {'), + bootstrapWorkflow.indexOf('\n roll_cell()', bootstrapWorkflow.indexOf('normalize_legacy_c3() {')) + ) + const trapInstalled = normalize.indexOf('trap restore_legacy_c3_fallback EXIT') + const isolated = normalize.indexOf('--mode isolate', trapInstalled) + const recreated = normalize.indexOf('recreate-instances', isolated) + const restored = normalize.indexOf('--mode restore', recreated) + const c3Verified = normalize.indexOf('staging-gce-c3 general', restored) + const restoreDisabled = normalize.indexOf('legacy_c3_isolated=false', c3Verified) + const trapCleared = normalize.indexOf('trap - EXIT', restoreDisabled) + assert.ok( + trapInstalled < isolated && + isolated < recreated && + recreated < restored && + restored < c3Verified && + c3Verified < restoreDisabled && + restoreDisabled < trapCleared + ) + assert.equal(normalize.indexOf('trap - EXIT', trapInstalled), trapCleared) + assert.match(bootstrapWorkflow, /--heartbeat either/) + assert.doesNotMatch(bootstrapWorkflow, /heartbeat=stale/) + assert.match( + rollCell, + /"\$\{desired_cap\}" "\$\{desired_bound\}" absent-or-stale[\s\S]*?deploy-relay-blue-green\.mjs[\s\S]*?"\$\{desired_cap\}" "\$\{desired_bound\}"/ + ) +}) + +test('workflows read desired topology from configuration and gate exact predecessors', () => { + assert.match(workflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/) + assert.match(bootstrapWorkflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/) + assert.match(workflow, /1000\/0\)[\s\S]*?PREDECESSOR_C3_CAP=600/) + assert.match(workflow, /1000\/60\)[\s\S]*?PREDECESSOR_C3_BOUND=0/) + assert.match(workflow, /600\/60\)[\s\S]*?PREDECESSOR_C3_CAP=1000/) + assert.match(workflow, /Unsupported staging capacity transition/) +}) + +test('capacity apply resumes after director or cell success and preserves the no-op restart proof', () => { + for (const phase of ['predecessor', 'director-ready', 'cell-ready', 'cell-active']) { + assert.match(workflow, new RegExp(`TRANSITION_PHASE=${phase}`)) + } + assert.match(workflow, /--argjson expected "\$\{PREDECESSOR_CELLS_JSON\}"/) + assert.match(workflow, /--argjson expected "\$\{DESIRED_CELLS_JSON\}"/) + assert.match( + workflow, + /test "\$\{TRANSITION_PHASE\}" = cell-ready; then[\s\S]*?test "\$\{CELL_PLAN_CHANGES\}" = 0/ + ) + assert.match( + workflow, + /test "\$\{TRANSITION_PHASE\}" = cell-active; then[\s\S]*?recreate_fixed_one_instance/ + ) + assert.match(workflow, /--admission migration-only/) +}) diff --git a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs new file mode 100644 index 00000000000..fece62f9ea2 --- /dev/null +++ b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs @@ -0,0 +1,228 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs' +import { + RELAY_WORKFLOW_FILE_PREFIX, + relayWorkflowFile, + relayWorkflowPath +} from './relay-repository.mjs' + +const identity = readFileSync( + new URL('../../infra/terraform/relay-staging-deploy-iam.tf', import.meta.url), + 'utf8' +) +const shared = readFileSync( + new URL('../../infra/terraform/relay-shared.tf', import.meta.url), + 'utf8' +) +const variables = readFileSync( + new URL('../../infra/terraform/variables.tf', import.meta.url), + 'utf8' +) +const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8') +const stagingTfvars = readFileSync( + new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url), + 'utf8' +) + +// The exact five staging Relay workflows the relay-owned deploy identity serves. +const DEPLOY_WORKFLOWS = [ + 'bootstrap-relay-staging-capacity.yml', + 'deploy-relay-staging-gce-candidate.yml', + 'deploy-relay-staging.yml', + 'operate-relay-asia-admission.yml', + 'power-relay-staging.yml' +] + +const GENERIC_STAGING_PAIR = + /vars\.STAGING_GCP_(?:WORKLOAD_IDENTITY_PROVIDER|DEPLOY_SERVICE_ACCOUNT)\b/ + +const workflowNames = workflowFiles +// DEPLOY_WORKFLOWS holds the names Terraform pins; the files on disk carry the copy's prefix. +const workflow = (name) => readWorkflow(relayWorkflowFile(name)) + +function block(type, name) { + const start = identity.indexOf(`resource "${type}" "${name}"`) + assert.notEqual(start, -1, `${type}.${name} is missing`) + const next = identity.indexOf('\nresource "', start + 1) + return identity.slice(start, next === -1 ? undefined : next) +} + +function declaredFamilies() { + return [...identity.matchAll(/^resource "([a-z0-9_]+)" "([a-z0-9_]+)"/gm)] + .map((match) => `${match[1]}.${match[2]}`) + .sort() +} + +function providerWorkflowFiles() { + const match = identity.match( + /github_staging_relay_deploy_workflow_files\s*=\s*\[([\s\S]*?)\n {2}\]/ + ) + assert.ok(match, 'github_staging_relay_deploy_workflow_files is missing') + return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1]) +} + +function variableDefault(name) { + const match = variables.match( + new RegExp(`variable "${name}" \\{[\\s\\S]*?default\\s*=\\s*"([^"]*)"`) + ) + assert.ok(match, `variable ${name} has no default`) + return match[1] +} + +test('no Relay workflow authenticates as the shared staging deploy identity', () => { + for (const name of workflowNames()) { + if (!name.includes('relay')) continue + assert.doesNotMatch(readWorkflow(name), GENERIC_STAGING_PAIR, name) + } +}) + +test('the five staging Relay workflows name the relay deploy pair', () => { + for (const name of DEPLOY_WORKFLOWS) { + const source = workflow(name) + assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/, name) + assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/, name) + } +}) + +// Why: the Asia workflow serves both environments from one job. Repointing its staging arm must +// not move production off the relay-owned shared account. +test('the Asia admission production arm keeps the production deploy pair', () => { + const source = workflow('operate-relay-asia-admission.yml') + assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/) + assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/) +}) + +test('the provider allowlists exactly those five workflow refs', () => { + const files = providerWorkflowFiles() + assert.deepEqual([...files].sort(), [...DEPLOY_WORKFLOWS].sort()) + for (const file of files) { + assert.match(file, /^[a-z0-9-]+\.yml$/) + } + // Each accepted repository turns that file list into its own exact refs. + assert.match( + identity, + /for workflow_file in local\.github_staging_relay_deploy_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/ + ) + + const provider = block( + 'google_iam_workload_identity_pool_provider', + 'github_staging_relay_deploy' + ) + assert.match(provider, /workload_identity_pool_provider_id\s*=\s*"github-relay-deploy"/) + assert.match(provider, /concat\(local\.relay_github_leading_repository_claims/) + assert.match(provider, /assertion\.ref == 'refs\/heads\/main'/) + assert.match(provider, /assertion\.environment == 'staging'/) + assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_deploy"\]/) + // A prefix match would turn the allowlist into a namespace grant with no Terraform diff. + assert.doesNotMatch(provider, /startsWith|endsWith/) +}) + +// Why: the documented attribute_condition limit is 4096 characters and the expression grows with +// every workflow added. Render it the way Terraform does and keep the headroom visible. +test('the rendered attribute condition stays inside the provider limit', () => { + const repository = `${variableDefault('github_owner')}/${variableDefault('github_repo')}` + const claims = [ + `assertion.repository == '${repository}'`, + `assertion.repository_id == '${variableDefault('github_repo_id')}'`, + `assertion.repository_owner_id == '${variableDefault('github_owner_id')}'` + ] + const workflowRefs = providerWorkflowFiles().map( + (file) => `${repository}/${relayWorkflowPath(file)}@refs/heads/main` + ) + const rendered = [ + ...claims, + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'staging'", + `(${workflowRefs.map((ref) => `assertion.workflow_ref == '${ref}'`).join(' || ')})` + ].join(' && ') + assert.ok(rendered.length < 4096, `rendered condition is ${rendered.length} characters`) + // 797 is the private repository's rendered length. This copy prefixes every workflow filename, + // which is the only difference, so the pin still moves the moment a workflow is added or dropped. + assert.equal(rendered.length, 797 + workflowRefs.length * RELAY_WORKFLOW_FILE_PREFIX.length) +}) + +// Why: the census is the point. A binding added here without a workflow step behind it, or one +// silently dropped, changes what the staging Relay credential can reach. +test('the staging deploy identity declares exactly its enumerated grants', () => { + assert.deepEqual(declaredFamilies(), [ + 'google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader', + 'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer', + 'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer', + 'google_iam_workload_identity_pool_provider.github_staging_relay_deploy', + 'google_project_iam_member.github_staging_relay_deploy_compute_viewer', + 'google_service_account.github_staging_relay_deploy', + 'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user', + 'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user', + 'google_storage_bucket_iam_member.github_staging_relay_deploy_state', + 'google_storage_bucket_iam_member.github_staging_relay_deploy_state_list' + ]) + // Each grant carries a comment naming the workflow step that needs it; the account, its + // provider, and the pool binding are the identity itself and are covered by the file header. + const identityFamilies = new Set([ + 'google_service_account.github_staging_relay_deploy', + 'google_iam_workload_identity_pool_provider.github_staging_relay_deploy', + 'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user' + ]) + for (const family of declaredFamilies()) { + if (identityFamilies.has(family)) continue + const [type, name] = family.split('.') + const preceding = identity.slice(0, identity.indexOf(`resource "${type}" "${name}"`)).trimEnd() + assert.match(preceding.slice(preceding.lastIndexOf('\n') + 1), /^#/, `${family} has no justifying comment`) + } + assert.match(identity, /var\.environment == "staging"/) + + const state = block('google_storage_bucket_iam_member', 'github_staging_relay_deploy_state') + assert.match(state, /roles\/storage\.objectViewer/) + assert.match(state, /objects\/terraform\/state\/default\.tfstate/) + assert.match(state, /objects\/terraform\/state\/default\.tflock/) + assert.doesNotMatch(state, /resource\.name\.startsWith/) + assert.doesNotMatch(state, /objectAdmin/) + + const director = block( + 'google_cloud_run_v2_service_iam_member', + 'github_staging_relay_deploy_director_developer' + ) + assert.match(director, /name\s*=\s*var\.relay_cloud_run_service_name/) + + // Project-wide run.developer or artifactregistry.writer would let the staging Relay credential + // deploy the API service or push images; the shared account holds both today. + const projectRoles = declaredFamilies() + .filter((family) => family.startsWith('google_project_iam_member.')) + .map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1]) + assert.deepEqual(projectRoles, ['roles/compute.viewer']) + assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/) + assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/) +}) + +// Why: the auth-plane grants are guarded on a variable, so an unset tfvars entry would drop them +// silently and Power Relay Staging would fail only on the sleep path. +test('staging pins the shared auth service the power workflow scales', () => { + assert.match(stagingTfvars, /relay_staging_power_auth_service_name\s*=\s*"orca-cloud-auth-staging"/) + assert.match(variables, /variable "relay_staging_power_auth_service_name"/) + for (const name of [ + 'github_staging_relay_deploy_auth_developer', + 'github_staging_relay_deploy_auth_runtime_user' + ]) { + const type = name.endsWith('runtime_user') + ? 'google_service_account_iam_member' + : 'google_cloud_run_v2_service_iam_member' + assert.match(block(type, name), /var\.relay_staging_power_auth_service_name != ""/) + } +}) + +// Why: flipping this local is what moves the staging cells' startup metadata and the director's +// ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT onto the new account. Production must keep the shared one. +test('the deploy account email is environment-conditional', () => { + assert.match( + shared, + /relay_github_deploy_service_account_email = \(\s*var\.environment == "production"\s*\? "\$\{var\.name_prefix\}-gha-deploy@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*: "\$\{var\.name_prefix\}-gha-relay@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*\)/ + ) + assert.match(identity, /account_id\s*=\s*"\$\{var\.name_prefix\}-gha-relay"/) +}) + +test('the identity is exposed through its own outputs', () => { + assert.match(outputs, /output "github_staging_relay_deploy_workload_identity_provider"/) + assert.match(outputs, /output "github_staging_relay_deploy_service_account"/) +}) diff --git a/cloud/dev/scripts/render-workload-identity-conditions.mjs b/cloud/dev/scripts/render-workload-identity-conditions.mjs new file mode 100644 index 00000000000..9a7c2e5cc09 --- /dev/null +++ b/cloud/dev/scripts/render-workload-identity-conditions.mjs @@ -0,0 +1,535 @@ +// Renders every Workload Identity provider `attribute_condition` exactly as +// Terraform would, so contract tests can pin the resulting strings without a +// plan. Understands only the HCL subset those expressions use. +// +// Each root is loaded on its own: the relay and apps roots both declare a provider named +// `github` while the staging copy waits on its state surgery, and only separate scopes can +// show that the two render the same string. +// +// Only the relay root ships in this repository. The apps root is still declared so this stays a +// straight copy of the private original, and is skipped when its directory is absent. +import { existsSync } from 'node:fs' +import { readFile } from 'node:fs/promises' + +const TERRAFORM_ROOTS = { + relay: { + directory: 'infra/terraform', + sources: [ + 'infra/terraform/relay-shared.tf', + 'infra/terraform/relay-github-workflow-trust.tf', + 'infra/terraform/relay-github-actions.tf', + 'infra/terraform/relay-staging-deploy-iam.tf', + 'infra/terraform/relay-asia-topology-iam.tf', + 'infra/terraform/relay-asia-proof-iam.tf' + ] + }, + apps: { + directory: 'infra/terraform-apps', + sources: ['infra/terraform-apps/github-actions.tf'] + } +} + +export function hasTerraformRoot(root) { + const directory = TERRAFORM_ROOTS[root]?.directory + return directory !== undefined && existsSync(repoFile(directory)) +} + +export const TERRAFORM_ROOT_NAMES = Object.keys(TERRAFORM_ROOTS).filter(hasTerraformRoot) + +const PROVIDER_RESOURCE = 'google_iam_workload_identity_pool_provider' + +function repoFile(path) { + return new URL(`../../${path}`, import.meta.url) +} + +function skipTrivia(src, index) { + let i = index + for (;;) { + while (i < src.length && /\s/.test(src[i])) i += 1 + if (src[i] === '#') { + while (i < src.length && src[i] !== '\n') i += 1 + continue + } + return i + } +} + +// Returns the index just past the closing quote of the string starting at `i`. +function endOfString(src, i) { + let cursor = i + 1 + while (src[cursor] !== '"') { + if (src[cursor] === '\\') { + cursor += 2 + continue + } + if (src[cursor] === '$' && src[cursor + 1] === '{') { + cursor = endOfInterpolation(src, cursor + 2).next + continue + } + cursor += 1 + } + return cursor + 1 +} + +function endOfInterpolation(src, i) { + let depth = 1 + let cursor = i + while (depth > 0) { + const char = src[cursor] + if (char === undefined) throw new Error('unterminated interpolation') + if (char === '"') { + cursor = endOfString(src, cursor) + continue + } + if (char === '{') depth += 1 + else if (char === '}') { + depth -= 1 + if (depth === 0) break + } + cursor += 1 + } + return { text: src.slice(i, cursor), next: cursor + 1 } +} + +// Stands in for a loop variable when the collection is empty: the body still has to be parsed +// once to find where it ends, and any attribute of the probe is another probe. +const PROBE = new Proxy( + {}, + { + get: (target, key) => (key === Symbol.toPrimitive ? () => '' : PROBE) + } +) + +function readMember(value, key) { + if (value === PROBE) return PROBE + if (value === null || value === undefined) throw new Error(`cannot read ${String(key)} of ${value}`) + if (Array.isArray(value)) { + if (typeof key !== 'number') throw new Error(`list index must be a number, got ${String(key)}`) + if (!Number.isInteger(key) || key < 0 || key >= value.length) { + throw new Error(`list index ${key} is out of range`) + } + return value[key] + } + if (typeof value !== 'object') throw new Error(`cannot index ${typeof value}`) + if (!Object.hasOwn(value, key)) throw new Error(`unknown attribute ${String(key)}`) + return value[key] +} + +// [key, value] pairs the way HCL iterates: list index and element, or object key and value. +function collectionEntries(collection) { + if (Array.isArray(collection)) return collection.map((item, index) => [index, item]) + if (collection && typeof collection === 'object') return Object.entries(collection) + throw new Error(`cannot iterate ${typeof collection}`) +} + +class ExpressionParser { + constructor(source, scope) { + this.source = source + this.scope = scope + this.index = 0 + } + + parse() { + const value = this.parseTernary() + this.index = skipTrivia(this.source, this.index) + if (this.index !== this.source.length) { + throw new Error(`trailing expression text: ${this.source.slice(this.index)}`) + } + return value + } + + peek(token) { + this.index = skipTrivia(this.source, this.index) + return this.source.startsWith(token, this.index) + } + + eat(token) { + if (!this.peek(token)) return false + this.index += token.length + return true + } + + expect(token) { + if (!this.eat(token)) { + throw new Error(`expected ${token} at ${this.source.slice(this.index, this.index + 40)}`) + } + } + + parseTernary() { + const condition = this.parseOr() + if (!this.eat('?')) return condition + const consequent = this.parseTernary() + this.expect(':') + const alternate = this.parseTernary() + return condition ? consequent : alternate + } + + parseOr() { + let left = this.parseAnd() + while (this.eat('||')) left = Boolean(this.parseAnd()) || Boolean(left) + return left + } + + parseAnd() { + let left = this.parseEquality() + while (this.eat('&&')) left = Boolean(this.parseEquality()) && Boolean(left) + return left + } + + parseEquality() { + let left = this.parseUnary() + for (;;) { + if (this.eat('==')) left = left === this.parseUnary() + else if (this.eat('!=')) left = left !== this.parseUnary() + else return left + } + } + + parseUnary() { + return this.parsePostfix(this.parsePrimary()) + } + + parsePrimary() { + if (this.eat('(')) { + const value = this.parseTernary() + this.expect(')') + return value + } + if (this.peek('"')) return this.parseString() + if (this.peek('[')) return this.parseList() + if (this.peek('{')) return this.parseObject() + const number = /^[0-9]+/.exec(this.source.slice(this.index)) + if (number) { + this.index += number[0].length + return Number(number[0]) + } + return this.parseIdentifier() + } + + parsePostfix(value) { + let current = value + for (;;) { + if (this.eat('.')) { + current = readMember(current, this.readWord()) + continue + } + if (this.peek('[')) { + this.index += 1 + const key = this.parseTernary() + this.expect(']') + current = readMember(current, key) + continue + } + return current + } + } + + // `for a in x : body` / `for a, b in x : body`, shared by list and object comprehensions. + parseComprehension(readBody) { + const names = [this.readWord()] + if (this.eat(',')) names.push(this.readWord()) + this.expect('in') + const collection = this.parseUnary() + this.expect(':') + const bodyStart = skipTrivia(this.source, this.index) + const entries = collectionEntries(collection) + const bodyParser = ([key, item]) => { + const bindings = { ...this.scope.bindings } + if (names.length === 1) bindings[names[0]] = Array.isArray(collection) ? item : key + else { + bindings[names[0]] = key + bindings[names[1]] = item + } + const parser = new ExpressionParser(this.source, { ...this.scope, bindings }) + parser.index = bodyStart + return parser + } + // Parse once with a probe binding to find where the body ends, because an empty + // collection would never parse it. + const probe = bodyParser(entries[0] ?? [PROBE, PROBE]) + readBody(probe) + this.index = probe.index + return entries.map((entry) => readBody(bodyParser(entry))) + } + + parseObject() { + this.expect('{') + if (this.eat('for')) { + const pairs = this.parseComprehension((parser) => { + const key = parser.parseTernary() + parser.expect('=>') + return [key, parser.parseTernary()] + }) + this.expect('}') + return Object.fromEntries(pairs) + } + const object = {} + if (this.eat('}')) return object + for (;;) { + const key = this.peek('"') ? this.parseString() : this.readWord() + this.expect('=') + object[key] = this.parseTernary() + this.eat(',') + if (this.eat('}')) return object + } + } + + parseString() { + this.index = skipTrivia(this.source, this.index) + const src = this.source + let cursor = this.index + 1 + let rendered = '' + while (src[cursor] !== '"') { + if (src[cursor] === '\\') { + rendered += src[cursor + 1] + cursor += 2 + continue + } + if (src[cursor] === '$' && src[cursor + 1] === '{') { + const { text, next } = endOfInterpolation(src, cursor + 2) + rendered += String(evaluate(text, this.scope)) + cursor = next + continue + } + rendered += src[cursor] + cursor += 1 + } + this.index = cursor + 1 + return rendered + } + + parseList() { + this.expect('[') + if (this.eat('for')) { + const items = this.parseComprehension((parser) => parser.parseTernary()) + this.expect(']') + return items + } + const items = [] + if (this.eat(']')) return items + for (;;) { + items.push(this.parseTernary()) + if (this.eat(',')) { + if (this.eat(']')) return items + continue + } + this.expect(']') + return items + } + } + + readWord() { + this.index = skipTrivia(this.source, this.index) + const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(this.source.slice(this.index)) + if (!match) throw new Error(`expected identifier at ${this.source.slice(this.index, this.index + 40)}`) + this.index += match[0].length + return match[0] + } + + parseIdentifier() { + const word = this.readWord() + if (word === 'join') { + this.expect('(') + const separator = this.parseTernary() + this.expect(',') + const parts = this.parseTernary() + this.eat(',') + this.expect(')') + return parts.join(separator) + } + if (word === 'concat') { + this.expect('(') + const lists = [] + for (;;) { + lists.push(this.parseTernary()) + if (this.eat(',')) { + if (this.eat(')')) break + continue + } + this.expect(')') + break + } + return lists.flat() + } + if (word === 'length') { + this.expect('(') + const value = this.parseTernary() + this.eat(',') + this.expect(')') + return collectionEntries(value).length + } + if (word === 'local') { + this.expect('.') + return resolveLocal(this.readWord(), this.scope) + } + if (word === 'var') { + this.expect('.') + const name = this.readWord() + if (!(name in this.scope.variables)) throw new Error(`unknown variable ${name}`) + return this.scope.variables[name] + } + if (word in this.scope.bindings) return this.scope.bindings[word] + if (word === 'true') return true + if (word === 'false') return false + throw new Error(`unsupported identifier ${word}`) + } +} + +function evaluate(source, scope) { + return new ExpressionParser(source, scope).parse() +} + +function resolveLocal(name, scope) { + if (scope.resolved.has(name)) return scope.resolved.get(name) + if (!scope.locals.has(name)) throw new Error(`unknown local ${name}`) + if (scope.resolving.has(name)) throw new Error(`local cycle at ${name}`) + scope.resolving.add(name) + const value = evaluate(scope.locals.get(name), { ...scope, bindings: {} }) + scope.resolving.delete(name) + scope.resolved.set(name, value) + return value +} + +function collectLocals(source, locals) { + const blockPattern = /^locals \{$/gm + let match + while ((match = blockPattern.exec(source)) !== null) { + const end = source.indexOf('\n}\n', match.index) + const body = source.slice(match.index + match[0].length, end) + let name = null + let buffer = [] + const flush = () => { + if (name) locals.set(name, buffer.join('\n')) + } + for (const line of body.split('\n')) { + const assignment = /^ {2}([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line) + if (assignment) { + flush() + name = assignment[1] + buffer = [assignment[2]] + continue + } + if (name && line.trim() !== '' && !line.trim().startsWith('#')) buffer.push(line) + } + flush() + } +} + +function collectProviderFields(source, fields) { + const pattern = new RegExp(`resource "${PROVIDER_RESOURCE}" "([A-Za-z_0-9]+)" \\{`, 'g') + let match + while ((match = pattern.exec(source)) !== null) { + const end = source.indexOf('\n}\n', match.index) + const body = source.slice(match.index, end) + const conditionStart = body.indexOf(' attribute_condition = ') + const conditionEnd = body.indexOf('\n\n oidc {', conditionStart) + const countStart = body.indexOf(' count = ') + fields.set(match[1], { + count: body.slice(countStart + ' count = '.length, body.indexOf('\n', countStart)), + condition: body.slice(conditionStart + ' attribute_condition = '.length, conditionEnd) + }) + } +} + +// Values that are not a plain quoted string (a list of objects, say) are read with the +// expression parser; anything it cannot evaluate is left undefined, exactly as before. +function parseValueAt(source, index) { + const parser = new ExpressionParser(source, { + locals: new Map(), + variables: {}, + bindings: {}, + resolved: new Map(), + resolving: new Set() + }) + parser.index = index + return parser.parseTernary() +} + +function collectVariableDefaults(source, variables) { + const pattern = /variable "([A-Za-z_0-9]+)" \{([\s\S]*?)\n\}/g + let match + while ((match = pattern.exec(source)) !== null) { + const body = match[2] + const fallback = /\n\s*default\s*=\s*"([^"]*)"/.exec(body) + if (fallback) { + variables[match[1]] = fallback[1] + continue + } + const assignment = /\n\s*default\s*=\s*/.exec(body) + if (!assignment) continue + const start = match.index + match[0].indexOf(body) + assignment.index + assignment[0].length + try { + variables[match[1]] = parseValueAt(source, start) + } catch { + // A default this evaluator does not understand is not one any condition reads. + } + } +} + +function collectTfvars(source, variables) { + let offset = 0 + for (const line of source.split('\n')) { + const quoted = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*"([^"]*)"\s*$/.exec(line) + if (quoted) { + variables[quoted[1]] = quoted[2] + offset += line.length + 1 + continue + } + const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?=[[{])/.exec(line) + if (structured) { + try { + variables[structured[1]] = parseValueAt(source, offset + structured[0].length) + } catch { + // Same as above: unreadable here means unread by every condition. + } + } + offset += line.length + 1 + } +} + +async function loadScope(root, environment) { + const { directory, sources } = TERRAFORM_ROOTS[root] ?? {} + if (!sources) throw new Error(`unknown terraform root ${root}`) + const locals = new Map() + const providers = new Map() + for (const path of sources) { + const source = await readFile(repoFile(path), 'utf8') + collectLocals(source, locals) + collectProviderFields(source, providers) + } + const variables = {} + collectVariableDefaults(await readFile(repoFile(`${directory}/variables.tf`), 'utf8'), variables) + collectTfvars( + await readFile(repoFile(`${directory}/environments/${environment}.tfvars`), 'utf8'), + variables + ) + return { + providers, + scope: { locals, variables, bindings: {}, resolved: new Map(), resolving: new Set() } + } +} + +// Rendered `attribute_condition` per provider that the given root creates in the environment. +export async function renderRootAttributeConditions(root, environment) { + const { providers, scope } = await loadScope(root, environment) + const rendered = {} + for (const [name, fields] of providers) { + if (evaluate(fields.count, scope) === 0) continue + rendered[name] = evaluate(fields.condition, scope) + } + return rendered +} + +// Every root's rendered conditions, keyed by root and then by provider. +export async function renderAttributeConditions(environment) { + const rendered = {} + for (const root of TERRAFORM_ROOT_NAMES) { + rendered[root] = await renderRootAttributeConditions(root, environment) + } + return rendered +} + +export async function readTerraformLocal(name, environment, root = 'relay') { + const { scope } = await loadScope(root, environment) + return resolveLocal(name, scope) +} diff --git a/cloud/dev/scripts/run-relay-load-model.mjs b/cloud/dev/scripts/run-relay-load-model.mjs new file mode 100644 index 00000000000..6915b8eb8c2 --- /dev/null +++ b/cloud/dev/scripts/run-relay-load-model.mjs @@ -0,0 +1,8 @@ +import { assertSpreadModel, modeledRelayLoad } from './relay-load-model.mjs' + +const counts = process.argv.slice(2).length > 0 ? process.argv.slice(2).map(Number) : [4_000, 10_000] +for (const count of counts) { + const model = modeledRelayLoad(count) + assertSpreadModel(model) + console.log(JSON.stringify(model)) +} diff --git a/cloud/dev/scripts/run-relay-recovery-wave-gate.mjs b/cloud/dev/scripts/run-relay-recovery-wave-gate.mjs new file mode 100644 index 00000000000..fcc58d3474c --- /dev/null +++ b/cloud/dev/scripts/run-relay-recovery-wave-gate.mjs @@ -0,0 +1,19 @@ +import { readFileSync, statSync } from 'node:fs' +import { evaluateRecoveryWaveReport } from './relay-recovery-wave-gate.mjs' + +const args = process.argv.slice(2) +if (args.length !== 2 || args[0] !== '--report') { + process.stderr.write('usage: pnpm load:relay:recovery-gate -- --report \n') + process.exitCode = 1 +} else { + try { + if (statSync(args[1]).size > 1024 * 1024) throw new Error('report exceeds 1 MiB') + const report = JSON.parse(readFileSync(args[1], 'utf8')) + const result = evaluateRecoveryWaveReport(report) + process.stdout.write(`${JSON.stringify(result)}\n`) + if (result.status !== 'PASS') process.exitCode = 1 + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } +} diff --git a/cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs b/cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs new file mode 100644 index 00000000000..1ac07841514 --- /dev/null +++ b/cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs @@ -0,0 +1,98 @@ +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +const MODES = new Set([ + 'inspect', 'initialize', 'verify', 'registered', 'register', + 'promote', 'recover-promotion', 'rollback' +]) +const STATES = new Set(['absent', 'existing-only', 'migration-only', 'general']) + +function validCellId(value) { + return typeof value === 'string' && value.length >= 1 && value.length <= 128 +} + +function cellIds(value, label) { + if ( + !Array.isArray(value) || value.length > 256 || + value.some((cellId) => !validCellId(cellId)) + ) { + throw new Error(`${label} is invalid`) + } + return [...value] +} + +function membership(value) { + if (value === undefined || value === null) return null + if (typeof value !== 'object' || Array.isArray(value)) { + throw new Error('membership is invalid') + } + return { + existingOnly: cellIds(value.existingOnly, 'existing-only membership'), + migrationOnly: cellIds(value.migrationOnly, 'migration-only membership'), + general: cellIds(value.general, 'general membership') + } +} + +function states(value) { + if (typeof value !== 'object' || Array.isArray(value)) throw new Error('states are invalid') + const entries = Object.entries(value) + if ( + entries.length === 0 || entries.length > 256 || + entries.some(([cellId, state]) => !validCellId(cellId) || !STATES.has(state)) + ) { + throw new Error('states are invalid') + } + return Object.fromEntries(entries) +} + +function optionalBoolean(value, label) { + if (value === undefined || value === null) return null + if (typeof value !== 'boolean') throw new Error(`${label} is invalid`) + return value +} + +export function sanitizeRelayAsiaAdmissionResult(input) { + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new Error('admission result must be an object') + } + if (!MODES.has(input.mode)) throw new Error('mode is invalid') + if (!Number.isSafeInteger(input.generation) || input.generation < 0) { + throw new Error('generation is invalid') + } + if ( + input.membershipSha256 !== undefined && input.membershipSha256 !== null && + !/^[a-f0-9]{64}$/.test(input.membershipSha256) + ) throw new Error('membership SHA-256 is invalid') + const sanitizedMembership = membership(input.membership) + if ( + input.mode === 'inspect' && + (sanitizedMembership === null || !/^[a-f0-9]{64}$/.test(input.membershipSha256 ?? '')) + ) throw new Error('inspect membership evidence is incomplete') + const recovered = optionalBoolean(input.recovered, 'recovered') + const promoted = optionalBoolean(input.promoted, 'promoted') + if (input.mode === 'recover-promotion' && promoted === null) { + throw new Error('promotion recovery evidence is incomplete') + } + return { + v: 1, + mode: input.mode, + generation: input.generation, + states: states(input.states), + membership: sanitizedMembership, + membershipSha256: input.membershipSha256 ?? null, + recovered, + promoted + } +} + +function main() { + try { + const input = JSON.parse(readFileSync(0, 'utf8')) + process.stdout.write(`${JSON.stringify(sanitizeRelayAsiaAdmissionResult(input))}\n`) + } catch { + console.error('invalid Relay Asia admission result') + process.exitCode = 1 + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) main() diff --git a/cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs b/cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs new file mode 100644 index 00000000000..2f1cf64a8cb --- /dev/null +++ b/cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs @@ -0,0 +1,120 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { test } from 'node:test' +import { sanitizeRelayAsiaAdmissionResult } from './sanitize-relay-asia-admission-result.mjs' + +const script = fileURLToPath(new URL('./sanitize-relay-asia-admission-result.mjs', import.meta.url)) +const expectedKeys = [ + 'v', 'mode', 'generation', 'states', 'membership', 'membershipSha256', 'recovered', 'promoted' +] + +test('preserves explicit false and emits only the admission evidence allowlist', () => { + const result = sanitizeRelayAsiaAdmissionResult({ + mode: 'verify', generation: 7, states: { 'staging-gce-c4': 'migration-only' }, + membership: { + existingOnly: [], migrationOnly: ['staging-gce-c4'], general: [], token: 'nested-secret' + }, + membershipSha256: 'a'.repeat(64), + recovered: false, promoted: false, + token: 'secret', credential: 'secret', userId: 'user', relayHostId: 'host', arbitrary: true + }) + + assert.deepEqual(Object.keys(result), expectedKeys) + assert.equal(result.recovered, false) + assert.equal(result.promoted, false) + assert.equal(JSON.stringify(result).includes('secret'), false) + assert.deepEqual(Object.keys(result.membership), ['existingOnly', 'migrationOnly', 'general']) + assert.equal('token' in result, false) + assert.equal('credential' in result, false) + assert.equal('userId' in result, false) + assert.equal('relayHostId' in result, false) + assert.equal('arbitrary' in result, false) +}) + +test('uses null for missing optional admission evidence', () => { + assert.deepEqual(sanitizeRelayAsiaAdmissionResult({ + mode: 'verify', generation: 0, states: { 'staging-gce-c4': 'absent' } + }), { + v: 1, + mode: 'verify', + generation: 0, + states: { 'staging-gce-c4': 'absent' }, + membership: null, + membershipSha256: null, + recovered: null, + promoted: null + }) +}) + +test('preserves valid historical cell IDs accepted by the director schema', () => { + const legacyCellId = 'legacy staging cell' + const result = sanitizeRelayAsiaAdmissionResult({ + mode: 'inspect', + generation: 0, + states: { [legacyCellId]: 'general' }, + membership: { existingOnly: [], migrationOnly: [], general: [legacyCellId] }, + membershipSha256: 'a'.repeat(64) + }) + + assert.deepEqual(result.states, { [legacyCellId]: 'general' }) + assert.deepEqual(result.membership.general, [legacyCellId]) +}) + +test('sanitizes stdin through the command-line entry point', () => { + const run = spawnSync(process.execPath, [script], { + input: JSON.stringify({ + mode: 'verify', generation: 0, states: { 'staging-gce-c4': 'absent' }, + recovered: false, token: 'secret' + }), + encoding: 'utf8' + }) + + assert.equal(run.status, 0, run.stderr) + const result = JSON.parse(run.stdout) + assert.deepEqual(Object.keys(result), expectedKeys) + assert.equal(result.recovered, false) + assert.equal(JSON.stringify(result).includes('secret'), false) +}) + +test('requires the evidence needed by every sequenced operation mode', () => { + const states = { 'staging-gce-c4': 'migration-only' } + const membership = { + existingOnly: [], migrationOnly: ['staging-gce-c4'], general: [] + } + const validByMode = { + inspect: { membership, membershipSha256: 'a'.repeat(64) }, + initialize: {}, + verify: {}, + registered: {}, + register: {}, + promote: {}, + 'recover-promotion': { promoted: false }, + rollback: {} + } + + for (const [mode, evidence] of Object.entries(validByMode)) { + assert.doesNotThrow(() => sanitizeRelayAsiaAdmissionResult({ + mode, generation: 1, states, ...evidence + })) + assert.throws(() => sanitizeRelayAsiaAdmissionResult({ mode, generation: 1, ...evidence })) + } + assert.throws(() => sanitizeRelayAsiaAdmissionResult({ + mode: 'inspect', generation: 1, states, membership + })) + assert.throws(() => sanitizeRelayAsiaAdmissionResult({ + mode: 'inspect', generation: 1, states, membershipSha256: 'a'.repeat(64) + })) + assert.throws(() => sanitizeRelayAsiaAdmissionResult({ + mode: 'recover-promotion', generation: 1, states + })) +}) + +test('rejects invalid stdin without echoing it', () => { + const run = spawnSync(process.execPath, [script], { input: '{secret', encoding: 'utf8' }) + + assert.equal(run.status, 1) + assert.equal(run.stdout, '') + assert.equal(run.stderr, 'invalid Relay Asia admission result\n') + assert.equal(run.stderr.includes('{secret'), false) +}) diff --git a/cloud/dev/scripts/smoke-relay.mjs b/cloud/dev/scripts/smoke-relay.mjs new file mode 100644 index 00000000000..71689680a94 --- /dev/null +++ b/cloud/dev/scripts/smoke-relay.mjs @@ -0,0 +1,211 @@ +import { + createHash, + createHmac, + createPrivateKey, + createPublicKey, + randomUUID +} from 'node:crypto' +import { readFileSync } from 'node:fs' +import { createRequire } from 'node:module' + +const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url)) + +const relayUrl = process.argv[2]?.replace(/\/$/, '') +if (!relayUrl) throw new Error('usage: smoke-relay.mjs ') + +const health = await fetch(`${relayUrl}/health`) +if (!health.ok || (await health.json()).ok !== true) { + throw new Error(`relay health failed: ${health.status}`) +} + +const accessToken = process.env.ORCA_RELAY_SMOKE_ACCESS_TOKEN +const authUrl = process.env.ORCA_RELAY_SMOKE_AUTH_URL?.replace(/\/$/, '') +const signingKeyFile = process.env.ORCA_RELAY_SMOKE_SIGNING_KEY_FILE +if (!authUrl || (!accessToken && !signingKeyFile)) { + console.log('relay health smoke passed; provide auth URL plus an access token or operator signing-key file for a splice round-trip') + process.exit(0) +} + +const nacl = requireFromRelay('tweetnacl') +const WebSocket = requireFromRelay('ws') +const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import( + requireFromRelay.resolve('@orca-cloud/relay-contract') +) + +function nextMessage(socket) { + return new Promise((resolve, reject) => { + const onMessage = (data) => { + cleanup() + try { + resolve(JSON.parse(data.toString())) + } catch (error) { + reject(error) + } + } + const onError = (error) => { + cleanup() + reject(error) + } + const onClose = (code, reason) => { + cleanup() + reject(new Error(`socket closed before message: ${code} ${reason.toString()}`)) + } + const cleanup = () => { + socket.off('message', onMessage) + socket.off('error', onError) + socket.off('close', onClose) + } + socket.once('message', onMessage) + socket.once('error', onError) + socket.once('close', onClose) + }) +} + +function opened(socket) { + return new Promise((resolve, reject) => { + socket.once('open', resolve) + socket.once('error', reject) + }) +} + +const hostKeys = nacl.box.keyPair() +const relayHostId = createHash('sha256') + .update(hostKeys.publicKey) + .digest('base64url') + .slice(0, 16) +let relayToken +if (accessToken) { + const tokenResponse = await fetch(`${authUrl}/v1/desktop/auth/relay-token`, { + method: 'POST', + headers: { + authorization: `Bearer ${accessToken}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ + relayHostId, + hostPublicKeyB64: Buffer.from(hostKeys.publicKey).toString('base64') + }) + }) + if (!tokenResponse.ok) throw new Error(`relay-token exchange failed: ${tokenResponse.status}`) + ;({ relayToken } = await tokenResponse.json()) +} else { + // This operator-only path proves the deployed data plane before desktop UI + // exists; possession of the auth signing key remains the security boundary. + const { SignJWT } = await import(requireFromRelay.resolve('jose')) + const privateKey = createPrivateKey(readFileSync(signingKeyFile, 'utf8')) + const keyId = createHash('sha256') + .update(createPublicKey(privateKey).export({ type: 'spki', format: 'der' })) + .digest('base64url') + .slice(0, 16) + relayToken = await new SignJWT({ + prof: 'staging-smoke-profile', + org: 'staging-smoke-org', + purpose: 'host-control', + relayHostId + }) + .setProtectedHeader({ alg: 'ES256', kid: keyId }) + .setIssuer(authUrl) + .setAudience('orca-relay') + .setSubject('staging-smoke-user') + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey) +} +if (!relayToken) throw new Error('relay token was not produced') + +const wsOrigin = relayUrl.replace(/^http/, 'ws') +const control = new WebSocket(`${wsOrigin}/v1/host/control`, { + headers: { authorization: `Bearer ${relayToken}` }, + perMessageDeflate: false +}) +await opened(control) +control.send( + JSON.stringify({ + type: 'host-hello', + v: 1, + relayHostId, + assignmentEpoch: 1, + hostPublicKeyB64: Buffer.from(hostKeys.publicKey).toString('base64'), + appVersion: 'relay-smoke' + }) +) +const challenge = await nextMessage(control) +const plaintext = nacl.box.open( + Buffer.from(challenge.ciphertextB64, 'base64'), + Buffer.from(challenge.nonceB64, 'base64'), + Buffer.from(challenge.relayEphemeralPublicKeyB64, 'base64'), + hostKeys.secretKey +) +if (!plaintext) throw new Error('host proof challenge did not decrypt') +const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`) +const transcriptLength = new DataView( + plaintext.buffer, + plaintext.byteOffset + domain.length, + 4 +).getUint32(0, false) +const transcriptStart = domain.length + 4 +const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength) +const secret = plaintext.slice(transcriptStart + transcriptLength) +const proofB64 = createHmac('sha256', secret) + .update(buildHostProofMacInput(transcript)) + .digest('base64') +control.send(JSON.stringify({ + type: 'host-challenge-ack', + challengeId: challenge.challengeId, + proofB64 +})) +const hostAck = await nextMessage(control) + +const relayDeviceId = `smoke-${randomUUID()}` +const invitePromise = nextMessage(control) +control.send(JSON.stringify({ type: 'invite-create', reqId: randomUUID(), relayDeviceId })) +const invite = await invitePromise + +const phone = new WebSocket(`${wsOrigin}/v1/connect/${relayHostId}`, { perMessageDeflate: false }) +await opened(phone) +const connectionPromise = nextMessage(control) +phone.send(JSON.stringify({ + type: 'relay-auth', + v: 1, + mode: 'connect', + credential: invite.inviteToken +})) +const connection = await connectionPromise +const data = new WebSocket(`${wsOrigin}/v1/host/data/${connection.connId}`, { + perMessageDeflate: false +}) +await opened(data) +const phoneHelloPromise = nextMessage(phone) +data.send(JSON.stringify({ + type: 'host-data-auth', + v: 1, + connTicket: connection.connTicket, + generation: hostAck.generation +})) +const phoneHello = await phoneHelloPromise +if (phoneHello.ok !== true) throw new Error('relay rejected smoke splice') + +const phoneEcho = new Promise((resolve, reject) => { + phone.once('message', (bytes, binary) => resolve({ bytes: Buffer.from(bytes), binary })) + phone.once('error', reject) +}) +data.send(Buffer.from([0x4f, 0x52, 0x43, 0x41])) +const echoed = await phoneEcho +if (!echoed.binary || !echoed.bytes.equals(Buffer.from([0x4f, 0x52, 0x43, 0x41]))) { + throw new Error('relay splice changed binary payload or opcode') +} + +const dataEcho = new Promise((resolve, reject) => { + data.once('message', (bytes, binary) => resolve({ bytes: Buffer.from(bytes), binary })) + data.once('error', reject) +}) +phone.send('orca-relay-smoke') +const returned = await dataEcho +if (returned.binary || returned.bytes.toString() !== 'orca-relay-smoke') { + throw new Error('relay splice changed text payload or opcode') +} + +phone.close() +data.close() +control.close() +console.log('relay authenticated splice smoke passed') diff --git a/cloud/dev/scripts/staging-relay-apply-guard.mjs b/cloud/dev/scripts/staging-relay-apply-guard.mjs new file mode 100644 index 00000000000..7595a585769 --- /dev/null +++ b/cloud/dev/scripts/staging-relay-apply-guard.mjs @@ -0,0 +1,51 @@ +import { execFileSync } from 'node:child_process' + +const PROJECT = 'onorca-cloud-staging' +const SQL_INSTANCE = 'orca-cloud-staging-auth-db' +const MIG_PREFIX = 'orca-cloud-staging-relay-gce-' + +function defaultGcloud(args) { + return execFileSync('gcloud', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'] + }).trim() +} + +export function stagingRelayPowerState(gcloud = defaultGcloud) { + const sqlActivationPolicy = gcloud([ + 'sql', + 'instances', + 'describe', + SQL_INSTANCE, + '--project', + PROJECT, + '--format=value(settings.activationPolicy)' + ]) + const groups = JSON.parse( + gcloud([ + 'compute', + 'instance-groups', + 'managed', + 'list', + '--project', + PROJECT, + `--filter=name~'^${MIG_PREFIX}'`, + '--format=json(name,targetSize)' + ]) + ) + return { sqlActivationPolicy, groups } +} + +export function assertStagingRelayAwake(gcloud = defaultGcloud) { + const state = stagingRelayPowerState(gcloud) + const sleepingGroups = state.groups.filter((group) => Number(group.targetSize) !== 1) + if ( + state.sqlActivationPolicy !== 'ALWAYS' || + state.groups.length < 2 || + sleepingGroups.length > 0 + ) { + throw new Error( + 'Staging Relay is asleep or partially awake. Run the Power Relay Staging wake workflow before any staging Terraform apply.' + ) + } +} diff --git a/cloud/dev/scripts/staging-relay-apply-guard.test.mjs b/cloud/dev/scripts/staging-relay-apply-guard.test.mjs new file mode 100644 index 00000000000..8035ffaf3a4 --- /dev/null +++ b/cloud/dev/scripts/staging-relay-apply-guard.test.mjs @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + assertStagingRelayAwake, + stagingRelayPowerState +} from './staging-relay-apply-guard.mjs' + +function gcloud(policy, targetSizes) { + return (args) => + args[0] === 'sql' + ? policy + : JSON.stringify( + targetSizes.map((targetSize, index) => ({ + name: `orca-cloud-staging-relay-gce-c${index + 1}`, + targetSize + })) + ) +} + +test('reads and accepts a fully awake staging topology', () => { + const command = gcloud('ALWAYS', [1, 1, 1]) + assert.equal(stagingRelayPowerState(command).groups.length, 3) + assert.doesNotThrow(() => assertStagingRelayAwake(command)) +}) + +test('refuses Terraform apply while SQL or any staging cell is asleep', () => { + assert.throws(() => assertStagingRelayAwake(gcloud('NEVER', [0, 0, 0])), /wake workflow/) + assert.throws(() => assertStagingRelayAwake(gcloud('ALWAYS', [1, 0, 0])), /partially awake/) + assert.throws(() => assertStagingRelayAwake(gcloud('ALWAYS', [])), /partially awake/) +}) diff --git a/cloud/dev/scripts/terraform-root-partition.mjs b/cloud/dev/scripts/terraform-root-partition.mjs new file mode 100644 index 00000000000..07ada7f624c --- /dev/null +++ b/cloud/dev/scripts/terraform-root-partition.mjs @@ -0,0 +1,131 @@ +#!/usr/bin/env node +import { readdirSync, readFileSync, writeFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +// Why: the relay Terraform root is being carved into foundation / relay / apps. Until every +// family is assigned to exactly one root per environment, a state move can silently orphan or +// double-manage a resource. This file is the single authority; the test pins it to the .tf files. + +export const ROOTS = ['foundation', 'apps', 'relay'] +export const ENVIRONMENTS = ['production', 'staging'] + +// Each root is its own Terraform directory; a family is declared in exactly the roots that own +// it somewhere, so the directory is the authority the fixture is checked against. Only the relay +// directory ships here, so only relay declarations can be read back; the partition still names +// all three roots because it is the authority for the whole carve. +export const ROOT_DIRECTORIES = { + relay: 'infra/terraform/' +} + +export const DECLARING_ROOTS = Object.keys(ROOT_DIRECTORIES) + +export const fixturePath = fileURLToPath( + new URL('../fixtures/terraform-root-partition/families.json', import.meta.url) +) + +function rootDirectory(root) { + const relative = ROOT_DIRECTORIES[root] + if (!relative) throw new Error(`unknown root ${root}`) + return fileURLToPath(new URL(`../../${relative}`, import.meta.url)) +} + +export function declaredFamilies(root = 'relay') { + const directory = rootDirectory(root) + const families = new Map() + for (const entry of readdirSync(directory).filter((name) => name.endsWith('.tf')).sort()) { + const text = readFileSync(`${directory}${entry}`, 'utf8') + for (const match of text.matchAll(/^resource "([a-z0-9_]+)" "([a-z0-9_]+)"/gm)) { + const address = `${match[1]}.${match[2]}` + if (families.has(address)) throw new Error(`duplicate declaration ${address}`) + families.set(address, entry) + } + } + return families +} + +// A family may be declared in two roots at once (the environment-conditional ten), so ownership +// is per (root, environment) while declaration is per root. +export function declaredRootFamilies(root) { + return new Set(declaredFamilies(root).keys()) +} + +export function ownedRootFamilies(partition, root) { + const families = new Set(partition[root]) + for (const [family, owners] of Object.entries(partition.env_conditional)) { + if (Object.values(owners).includes(root)) families.add(family) + } + return families +} + +export function readPartition(path = fixturePath) { + return JSON.parse(readFileSync(path, 'utf8')) +} + +// Family address for a state entry: strips [index] / ["key"] and ignores data sources. +export function familyOf(stateAddress) { + return stateAddress.replace(/\[.*$/, '') +} + +export function rootFor(partition, family, environment) { + if (!ENVIRONMENTS.includes(environment)) throw new Error(`unknown environment ${environment}`) + const conditional = partition.env_conditional[family] + if (conditional) return conditional[environment] + for (const root of ROOTS) if (partition[root].includes(family)) return root + return undefined +} + +export function expectedRootFamilies(partition, root, environment) { + const families = new Set(partition[root]) + for (const [family, owners] of Object.entries(partition.env_conditional)) { + if (owners[environment] === root) families.add(family) + } + return families +} + +// Compares `terraform state list` output for one root against the partition. +export function auditStateList(partition, root, environment, stateList) { + const expected = expectedRootFamilies(partition, root, environment) + const orphans = new Set(partition.state_orphans[environment] ?? []) + const unexpected = [] + const seen = new Set() + for (const line of stateList.split('\n').map((entry) => entry.trim()).filter(Boolean)) { + if (orphans.has(line)) continue + if (line.startsWith('data.')) continue + const family = familyOf(line) + seen.add(family) + if (!expected.has(family)) unexpected.push(line) + } + return { unexpected, seen: [...seen].sort() } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const [command, root, environment, stateFile] = process.argv.slice(2) + const partition = readPartition() + if (command === 'audit') { + const { unexpected } = auditStateList(partition, root, environment, readFileSync(stateFile, 'utf8')) + if (unexpected.length > 0) { + process.stderr.write(`entries in ${root}/${environment} state outside its partition:\n`) + for (const line of unexpected) process.stderr.write(` ${line}\n`) + process.exitCode = 1 + } else { + process.stdout.write(`${root}/${environment}: state matches partition\n`) + } + } else if (command === 'list') { + for (const family of [...expectedRootFamilies(partition, root, environment)].sort()) { + process.stdout.write(`${family}\n`) + } + } else if (command === 'write-families') { + // Refreshes only the declared-family lists; ownership edits stay manual. + for (const name of DECLARING_ROOTS) { + for (const family of declaredRootFamilies(name)) { + if (rootFor(partition, family, 'production') === undefined) { + throw new Error(`unassigned family ${family}; add it to the fixture first`) + } + } + } + writeFileSync(fixturePath, `${JSON.stringify(partition, null, 2)}\n`) + } else { + process.stderr.write('usage: terraform-root-partition.mjs audit|list [state-list-file]\n') + process.exitCode = 2 + } +} diff --git a/cloud/dev/scripts/terraform-root-partition.test.mjs b/cloud/dev/scripts/terraform-root-partition.test.mjs new file mode 100644 index 00000000000..b92cb92366d --- /dev/null +++ b/cloud/dev/scripts/terraform-root-partition.test.mjs @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict' +import { existsSync, readdirSync, readFileSync } from 'node:fs' +import { test } from 'node:test' +import { + DECLARING_ROOTS, + ENVIRONMENTS, + ROOTS, + auditStateList, + declaredFamilies, + declaredRootFamilies, + expectedRootFamilies, + ownedRootFamilies, + readPartition, + rootFor +} from './terraform-root-partition.mjs' + +const partition = readPartition() +const declared = new Map(DECLARING_ROOTS.flatMap((root) => [...declaredFamilies(root)].map(([family, file]) => [family, `${root}/${file}`]))) + +test('every declared resource family is assigned to exactly one root per environment', () => { + for (const family of declared.keys()) { + for (const environment of ENVIRONMENTS) { + const owners = ROOTS.filter((root) => expectedRootFamilies(partition, root, environment).has(family)) + assert.deepEqual(owners, [rootFor(partition, family, environment)], `${family} in ${environment}`) + } + } +}) + +// Only the relay directory ships here, so only the families the partition assigns to a declaring +// root can be checked back against a .tf file. The full listing is still checked for duplicates. +test('the partition names no family that is not declared', () => { + const listed = [ + ...ROOTS.flatMap((root) => partition[root]), + ...Object.keys(partition.env_conditional) + ] + for (const root of DECLARING_ROOTS) { + for (const family of ownedRootFamilies(partition, root)) { + assert.ok(declared.has(family), `${family} is not declared`) + } + } + assert.equal(new Set(listed).size, listed.length, 'a family is listed twice') +}) + +test('environment-conditional families are owned by different roots per environment', () => { + for (const [family, owners] of Object.entries(partition.env_conditional)) { + assert.deepEqual(Object.keys(owners).sort(), [...ENVIRONMENTS].sort(), family) + assert.notEqual(owners.production, owners.staging, `${family} is not really conditional`) + } +}) + +// Why: this is the census that survives the carve. Filename prefixes stopped meaning anything +// once each root became its own directory, so ownership is checked against the directory that +// declares the family. A root declares exactly what it owns in at least one environment; the +// environment-conditional ten are therefore declared twice, once per complementary count. +test('each root declares exactly the families it owns in some environment', () => { + for (const root of DECLARING_ROOTS) { + assert.deepEqual( + [...declaredRootFamilies(root)].sort(), + [...ownedRootFamilies(partition, root)].sort(), + root + ) + } +}) + +// Why: the removed blocks were a guard for the config-first window between the carve and the two +// state surgeries. Both are done; a removed block that resurfaces would silently turn a stray apply +// from "destroy" into "forget" and hide a real ownership mistake. +test('the carved families are gone from the relay root and nothing is guarded by a removed block', () => { + const relay = declaredRootFamilies('relay') + for (const family of [...partition.foundation, ...partition.apps]) { + assert.ok(!relay.has(family), `${family} is still declared in the relay root`) + } + assert.ok(!existsSync(new URL('../../infra/terraform/relay-root-carve-removed.tf', import.meta.url))) + for (const file of readdirSync(new URL('../../infra/terraform/', import.meta.url))) { + if (!file.endsWith('.tf')) continue + const source = readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8') + assert.doesNotMatch(source, /^removed \{/m, `${file} declares a removed block`) + } +}) + +// Why: every binding on the shared deploy account follows that account (relay in production, +// apps in staging). Letting one drift back into foundation recreates the dependency cycle the +// split exists to remove: foundation would need the account while relay needs the pool. +test('foundation owns only what both other roots must be able to bootstrap against', () => { + assert.deepEqual(partition.foundation, [ + 'google_artifact_registry_repository.api', + 'google_iam_workload_identity_pool.github', + 'google_project_service.required', + 'google_project_service.sqladmin', + 'google_service_account.runtime', + 'google_sql_database_instance.auth', + 'google_storage_bucket_iam_member.cloud_sql_rollout_lease', + 'google_storage_bucket_iam_member.cloud_sql_rollout_lease_bucket_reader' + ]) +}) + +// Why: the two staging orphans were cleared by the runbook; the allowance is empty so a stray +// entry is reported instead of silently tolerated again. +test('audit reports entries outside the partition and no longer tolerates the staging orphans', () => { + const stateList = [ + 'google_project_service.required["run.googleapis.com"]', + 'google_secret_manager_secret_iam_member.runtime_artifact_write_secret_accessor[0]', + 'data.google_compute_image.relay_gce_cos[0]', + 'google_cloud_run_v2_service.relay' + ].join('\n') + assert.deepEqual(partition.state_orphans, { staging: [], production: [] }) + const foundation = auditStateList(partition, 'foundation', 'staging', stateList) + assert.deepEqual(foundation.unexpected, [ + 'google_secret_manager_secret_iam_member.runtime_artifact_write_secret_accessor[0]', + 'google_cloud_run_v2_service.relay' + ]) + const relay = auditStateList(partition, 'relay', 'staging', stateList) + assert.deepEqual(relay.unexpected, [ + 'google_project_service.required["run.googleapis.com"]', + 'google_secret_manager_secret_iam_member.runtime_artifact_write_secret_accessor[0]' + ]) +}) diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs new file mode 100644 index 00000000000..953a214156e --- /dev/null +++ b/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs @@ -0,0 +1,295 @@ +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +const REGION = 'asia-east2' +const CELL_SHAPES = { + production: { + domain: 'relay.onorca.dev', + project: 'onorca-cloud', + cells: { + 'production-gce-c27': 'asia-east2-a', + 'production-gce-c28': 'asia-east2-b', + 'production-gce-c29': 'asia-east2-c' + } + }, + staging: { + domain: 'relay-staging.onorca.dev', + project: 'onorca-cloud-staging', + cells: { 'staging-gce-c4': 'asia-east2-a' } + } +} + +function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['plan-json', 'environment', 'cell-ids', 'region', 'image']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (!(values.environment in CELL_SHAPES)) throw new Error('--environment is invalid') + const cells = values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean) + const expectedCells = Object.keys(CELL_SHAPES[values.environment].cells) + if (new Set(cells).size !== cells.length || JSON.stringify(cells.sort()) !== JSON.stringify(expectedCells.sort())) { + throw new Error('--cell-ids must be the exact reviewed Asia topology set') + } + if (values.region !== REGION) throw new Error('--region must be asia-east2') + const expectedImagePrefix = `us-central1-docker.pkg.dev/${CELL_SHAPES[values.environment].project}/orca-cloud/relay@sha256:` + if (!values.image.startsWith(expectedImagePrefix) || !/sha256:[a-f0-9]{64}$/.test(values.image)) { + throw new Error('--image must be the environment Relay image pinned by digest') + } + return { planJson: values['plan-json'], environment: values.environment, cells, image: values.image } +} + +function address(resource, key) { + return `${resource}[${JSON.stringify(key)}]` +} + +function actions(change) { + return change.change?.actions ?? [] +} + +function sameActions(change, expected) { + return JSON.stringify(actions(change)) === JSON.stringify(expected) +} + +function startupValue(script, name) { + return new RegExp(`printf '${name}=%s\\\\n' '([^']+)'`).exec(script)?.[1] +} + +function relayGceName(environment) { + return environment === 'production' ? 'orca-cloud-relay-gce' : 'orca-cloud-staging-relay-gce' +} + +function unknownOrMatches(value, predicate) { + return value === undefined || value === null || predicate(String(value)) +} + +function requireCellTemplate(change, config, cellId) { + const after = change.change.after + const script = after?.metadata_startup_script ?? '' + if ( + after?.machine_type !== 'e2-standard-4' || + after?.labels?.['orca-relay-cell'] !== cellId || + after?.labels?.['orca-relay-region'] !== REGION || + !unknownOrMatches( + after?.network_interface?.[0]?.subnetwork, + (value) => value.includes(`/regions/${REGION}/subnetworks/`) + ) || + (after?.network_interface?.[0]?.access_config?.length ?? 0) !== 0 || + startupValue(script, 'ORCA_RELAY_REGION') !== REGION || + startupValue(script, 'ORCA_RELAY_CELL_CAPACITY') !== '6000' || + startupValue(script, 'ORCA_RELAY_DATABASE_POOL_MAX') !== '10' || + startupValue(script, 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP') !== '3000' || + startupValue(script, 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND') !== '60' || + startupValue(script, 'ORCA_RELAY_IMAGE_DIGEST') !== config.image.split('@')[1] || + !script.includes(`docker pull '${config.image}'`) || + !script.trimEnd().includes(`'${config.image}'`) + ) throw new Error(`${change.address} does not have the reviewed Asia cell shape`) +} + +function requireCellManager(change, config, cellId) { + const after = change.change.after + const hostname = cellId.split('-').at(-1) + const version = after?.version?.[0] + if ( + after?.zone !== CELL_SHAPES[config.environment].cells[cellId] || + after?.target_size !== 1 || + after?.version?.length !== 1 || + version?.name !== 'primary' || + !unknownOrMatches(version?.instance_template, (value) => + value.includes( + `/global/instanceTemplates/${relayGceName(config.environment)}-${hostname}-` + )) || + after?.update_policy?.[0]?.replacement_method !== 'RECREATE' || + after?.update_policy?.[0]?.max_surge_fixed !== 0 || + after?.update_policy?.[0]?.max_unavailable_fixed !== 1 + ) throw new Error(`${change.address} does not have the reviewed fixed-one Asia MIG shape`) +} + +function requireCellBackend(change, config, cellId) { + const after = change.change.after + const backend = after?.backend?.[0] + const zone = CELL_SHAPES[config.environment].cells[cellId] + const hostname = cellId.split('-').at(-1) + const name = `${relayGceName(config.environment)}-${hostname}` + if ( + after?.timeout_sec !== 86_400 || + after?.connection_draining_timeout_sec !== 300 || + after?.load_balancing_scheme !== 'EXTERNAL_MANAGED' || + after?.protocol !== 'HTTP' || + after?.port_name !== 'relay' || + after?.session_affinity !== 'NONE' || + after?.health_checks?.length !== 1 || + !unknownOrMatches(after.health_checks[0], (value) => + value.endsWith(`/global/healthChecks/${relayGceName(config.environment)}-ready`)) || + after?.backend?.length !== 1 || + backend?.balancing_mode !== 'UTILIZATION' || + backend?.max_utilization !== 0.8 || + backend?.capacity_scaler !== 1 || + !unknownOrMatches(backend?.group, (value) => + value.endsWith(`/zones/${zone}/instanceGroups/${name}`)) + ) throw new Error(`${change.address} does not have the reviewed Asia backend shape`) +} + +function requireNetworkResource(change, config) { + const after = change.change.after + const networkSuffix = `/global/networks/${relayGceName(config.environment)}` + if (after?.region !== REGION) throw new Error(`${change.address} is outside asia-east2`) + if ( + change.address.startsWith('google_compute_subnetwork.') && + (after.ip_cidr_range !== '10.42.1.0/24' || + after.private_ip_google_access !== true || + after.stack_type !== 'IPV4_ONLY' || + !unknownOrMatches(after.network, (value) => value.endsWith(networkSuffix))) + ) throw new Error(`${change.address} does not have the reviewed Asia subnet shape`) + if ( + change.address.startsWith('google_compute_router.') && + !unknownOrMatches(after.network, (value) => value.endsWith(networkSuffix)) + ) throw new Error(`${change.address} does not have the reviewed Asia router shape`) + if ( + change.address.startsWith('google_compute_router_nat.') && + (after.nat_ip_allocate_option !== 'AUTO_ONLY' || + after.source_subnetwork_ip_ranges_to_nat !== 'LIST_OF_SUBNETWORKS' || + after.subnetwork?.length !== 1 || + !unknownOrMatches(after.subnetwork[0]?.name, (value) => + value.endsWith( + `/regions/${REGION}/subnetworks/${relayGceName(config.environment)}-${REGION}` + )) || + JSON.stringify(after.subnetwork[0]?.source_ip_ranges_to_nat) !== + JSON.stringify(['ALL_IP_RANGES'])) + ) throw new Error(`${change.address} does not have the reviewed Asia NAT shape`) +} + +function canonical(value) { + if (!Array.isArray(value)) return JSON.stringify(value ?? []) + return JSON.stringify([...value].sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)))) +} + +function normalizeDescription(value) { + return { ...value, description: value.description ?? '' } +} + +function normalizeMatcher(value) { + const apiPrefix = 'https://www.googleapis.com/compute/v1/' + const defaultService = value.default_service + return { + ...normalizeDescription(value), + default_service: typeof defaultService === 'string' && defaultService.startsWith(apiPrefix) + ? defaultService.slice(apiPrefix.length) + : defaultService + } +} + +function requireUrlMap(change, config) { + const before = change.change.before ?? {} + const after = change.change.after ?? {} + const permitted = new Set(['host_rule', 'path_matcher', 'fingerprint']) + const changed = new Set([...Object.keys(before), ...Object.keys(after)].filter( + (key) => JSON.stringify(before[key]) !== JSON.stringify(after[key]) + )) + if ([...changed].some((key) => !permitted.has(key))) { + throw new Error('shared URL map changes outside host routing') + } + const newHosts = new Set() + const newMatchers = new Set() + for (const cellId of config.cells) { + const hostname = cellId.split('-').at(-1) + const host = `${hostname}.${CELL_SHAPES[config.environment].domain}` + const hostRules = after.host_rule?.filter( + (rule) => + rule.hosts?.length === 1 && + rule.hosts[0] === host && + rule.path_matcher === `cell-${hostname}` + ) ?? [] + if (hostRules.length !== 1) { + throw new Error(`shared URL map has no exact host for ${cellId}`) + } + const matchers = after.path_matcher?.filter( + (matcher) => + matcher.name === `cell-${hostname}` && + unknownOrMatches(matcher.default_service, (value) => + value.endsWith( + `/global/backendServices/${relayGceName(config.environment)}-${hostname}` + )) + ) ?? [] + if (matchers.length !== 1) { + throw new Error(`shared URL map has no exact backend route for ${cellId}`) + } + newHosts.add(host) + newMatchers.add(`cell-${hostname}`) + } + const preservedHostRules = (after.host_rule ?? []).filter( + (rule) => !(rule.hosts?.length === 1 && newHosts.has(rule.hosts[0])) + ) + const preservedMatchers = (after.path_matcher ?? []).filter( + (matcher) => !newMatchers.has(matcher.name) + ) + if ( + sameActions(change, ['update']) && + canonical(preservedHostRules.map(normalizeDescription)) !== + canonical((before.host_rule ?? []).map(normalizeDescription)) || + sameActions(change, ['update']) && + canonical(preservedMatchers.map(normalizeMatcher)) !== + canonical((before.path_matcher ?? []).map(normalizeMatcher)) + ) { + throw new Error('shared URL map does not preserve every existing exact route') + } +} + +export function validateRelayAsiaTopologyPlan(plan, config) { + if (!Array.isArray(plan.resource_changes)) throw new Error('Terraform plan has no resource changes') + const required = new Map([ + [address('google_compute_subnetwork.relay_gce_additional', REGION), [['create'], ['no-op']]], + [address('google_compute_router.relay_gce_additional', REGION), [['create'], ['no-op']]], + [address('google_compute_router_nat.relay_gce_additional', REGION), [['create'], ['no-op']]], + ['google_compute_url_map.relay_gce[0]', [['update'], ['no-op']]] + ]) + for (const cellId of config.cells) { + required.set(address('google_compute_instance_template.relay_gce_cell', cellId), [['create'], ['no-op']]) + required.set(address('google_compute_instance_group_manager.relay_gce_cell', cellId), [['create'], ['no-op']]) + required.set(address('google_compute_backend_service.relay_gce_cell', cellId), [['create'], ['no-op']]) + } + const byAddress = new Map(plan.resource_changes.map((change) => [change.address, change])) + for (const [resourceAddress, allowedActions] of required) { + const change = byAddress.get(resourceAddress) + if (!change || !allowedActions.some((expected) => sameActions(change, expected))) { + throw new Error(`${resourceAddress} is absent or has an unreviewed topology action`) + } + const cellId = config.cells.find((candidate) => resourceAddress.endsWith(`[${JSON.stringify(candidate)}]`)) + if (resourceAddress.startsWith('google_compute_instance_template.') && cellId) { + requireCellTemplate(change, config, cellId) + } else if (resourceAddress.startsWith('google_compute_instance_group_manager.') && cellId) { + requireCellManager(change, config, cellId) + } else if (resourceAddress.startsWith('google_compute_backend_service.') && cellId) { + requireCellBackend(change, config, cellId) + } else if ( + resourceAddress.startsWith('google_compute_subnetwork.') || + resourceAddress.startsWith('google_compute_router.') || + resourceAddress.startsWith('google_compute_router_nat.') + ) { + requireNetworkResource(change, config) + } else if (resourceAddress === 'google_compute_url_map.relay_gce[0]') { + requireUrlMap(change, config) + } + } + const changes = plan.resource_changes.filter((change) => !actions(change).every( + (action) => action === 'no-op' || action === 'read' + )) + for (const change of changes) { + const allowedActions = required.get(change.address) + if (!allowedActions || !allowedActions.some((expected) => sameActions(change, expected))) { + throw new Error(`${change.address} has an unreviewed topology action`) + } + } + return { environment: config.environment, cells: config.cells, changes: changes.length } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const config = parseArguments(process.argv.slice(2)) + const plan = JSON.parse(readFileSync(config.planJson, 'utf8')) + console.log(JSON.stringify(validateRelayAsiaTopologyPlan(plan, config))) +} diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs new file mode 100644 index 00000000000..154030489f5 --- /dev/null +++ b/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { validateRelayAsiaTopologyPlan } from './validate-relay-asia-topology-plan.mjs' + +const image = `us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:${'a'.repeat(64)}` +const config = { environment: 'staging', cells: ['staging-gce-c4'], image } +const create = (address, after = {}) => ({ address, change: { actions: ['create'], after } }) +const script = [ + `printf 'ORCA_RELAY_REGION=%s\\n' 'asia-east2'`, + `printf 'ORCA_RELAY_CELL_CAPACITY=%s\\n' '6000'`, + `printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '10'`, + `printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`, + `printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, + `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${image.split('@')[1]}'`, + `docker pull '${image}'`, + `'${image}'` +].join('\n') +const resources = [ + create('google_compute_subnetwork.relay_gce_additional["asia-east2"]', { + region: 'asia-east2', ip_cidr_range: '10.42.1.0/24', private_ip_google_access: true, + stack_type: 'IPV4_ONLY', + network: 'projects/p/global/networks/orca-cloud-staging-relay-gce' + }), + create('google_compute_router.relay_gce_additional["asia-east2"]', { + region: 'asia-east2', network: 'projects/p/global/networks/orca-cloud-staging-relay-gce' + }), + create('google_compute_router_nat.relay_gce_additional["asia-east2"]', { + region: 'asia-east2', nat_ip_allocate_option: 'AUTO_ONLY', + source_subnetwork_ip_ranges_to_nat: 'LIST_OF_SUBNETWORKS', + subnetwork: [{ + name: 'projects/p/regions/asia-east2/subnetworks/orca-cloud-staging-relay-gce-asia-east2', + source_ip_ranges_to_nat: ['ALL_IP_RANGES'] + }] + }), + create('google_compute_instance_template.relay_gce_cell["staging-gce-c4"]', { + machine_type: 'e2-standard-4', + labels: { 'orca-relay-cell': 'staging-gce-c4', 'orca-relay-region': 'asia-east2' }, + network_interface: [{ + subnetwork: 'projects/p/regions/asia-east2/subnetworks/relay', access_config: [] + }], + metadata_startup_script: script + }), + create('google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]', { + zone: 'asia-east2-a', target_size: 1, + version: [{ + name: 'primary', + instance_template: 'projects/p/global/instanceTemplates/orca-cloud-staging-relay-gce-c4-abc' + }], + update_policy: [{ replacement_method: 'RECREATE', max_surge_fixed: 0, max_unavailable_fixed: 1 }] + }), + create('google_compute_backend_service.relay_gce_cell["staging-gce-c4"]', { + timeout_sec: 86_400, connection_draining_timeout_sec: 300, + load_balancing_scheme: 'EXTERNAL_MANAGED', protocol: 'HTTP', port_name: 'relay', + session_affinity: 'NONE', + health_checks: ['projects/p/global/healthChecks/orca-cloud-staging-relay-gce-ready'], + backend: [{ + balancing_mode: 'UTILIZATION', max_utilization: 0.8, capacity_scaler: 1, + group: 'projects/p/zones/asia-east2-a/instanceGroups/orca-cloud-staging-relay-gce-c4' + }] + }), + { + address: 'google_compute_url_map.relay_gce[0]', + change: { + actions: ['update'], + before: { host_rule: [], path_matcher: [], fingerprint: 'old' }, + after: { + host_rule: [{ + hosts: ['c4.relay-staging.onorca.dev'], path_matcher: 'cell-c4' + }], + path_matcher: [{ + name: 'cell-c4', + default_service: 'projects/p/global/backendServices/orca-cloud-staging-relay-gce-c4' + }], + fingerprint: null + } + } + } +] + +test('accepts the exact additive staging Asia topology', () => { + assert.deepEqual(validateRelayAsiaTopologyPlan({ resource_changes: resources }, config), { + environment: 'staging', cells: ['staging-gce-c4'], changes: 7 + }) +}) + +test('accepts an idempotent empty plan', () => { + const noChanges = structuredClone(resources).map((resource) => ({ + ...resource, + change: { + ...resource.change, + actions: ['no-op'], + before: structuredClone(resource.change.after) + } + })) + assert.equal(validateRelayAsiaTopologyPlan({ resource_changes: noChanges }, config).changes, 0) +}) + +test('rejects a plan that omits any required topology resource', () => { + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: resources.slice(1) }, config), + /absent or has an unreviewed topology action/ + ) +}) + +test('rejects any US or unrelated mutation', () => { + const plan = structuredClone(resources) + plan.push(create('google_compute_subnetwork.relay_gce[0]')) + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), + /unreviewed topology action/ + ) +}) + +test('rejects delete and replacement actions', () => { + for (const invalidActions of [['delete'], ['create', 'delete']]) { + const plan = structuredClone(resources) + plan[0].change.actions = invalidActions + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), + /unreviewed topology action/ + ) + } +}) + +test('rejects a cell with different limits or image', () => { + const plan = structuredClone(resources) + plan[3].change.after.metadata_startup_script = script.replace("'3000'", "'5000'") + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), + /reviewed Asia cell shape/ + ) +}) + +test('rejects shared URL-map changes outside exact host routing', () => { + const plan = structuredClone(resources) + plan[6].change.after.default_service = 'unreviewed' + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), + /outside host routing/ + ) +}) + +test('rejects removal of an existing exact route', () => { + const plan = structuredClone(resources) + plan[6].change.before.host_rule = [{ hosts: ['c1.relay-staging.onorca.dev'] }] + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), + /preserve every existing exact route/ + ) +}) + +test('accepts provider normalization of preserved route descriptions', () => { + const plan = structuredClone(resources) + const matcher = { + name: 'cell-c1', + description: '', + default_service: + 'https://www.googleapis.com/compute/v1/projects/p/global/backendServices/orca-cloud-staging-relay-gce-c1' + } + plan[6].change.before.host_rule = [{ + description: '', hosts: ['c1.relay-staging.onorca.dev'], path_matcher: 'cell-c1' + }] + plan[6].change.before.path_matcher = [matcher] + plan[6].change.after.host_rule.unshift({ + description: null, hosts: ['c1.relay-staging.onorca.dev'], path_matcher: 'cell-c1' + }) + plan[6].change.after.path_matcher.unshift({ + ...matcher, + description: null, + default_service: 'projects/p/global/backendServices/orca-cloud-staging-relay-gce-c1' + }) + assert.equal(validateRelayAsiaTopologyPlan({ resource_changes: plan }, config).changes, 7) +}) + +test('rejects a changed preserved route backend', () => { + const plan = structuredClone(resources) + plan[6].change.before.path_matcher = [{ + name: 'cell-c1', + default_service: + 'https://www.googleapis.com/compute/v1/projects/p/global/backendServices/orca-cloud-staging-relay-gce-c1' + }] + plan[6].change.after.path_matcher.unshift({ + name: 'cell-c1', + default_service: 'projects/p/global/backendServices/orca-cloud-staging-relay-gce-c2' + }) + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), + /preserve every existing exact route/ + ) +}) + +test('rejects a different Asia subnet range', () => { + const plan = structuredClone(resources) + plan[0].change.after.ip_cidr_range = '10.99.0.0/24' + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), + /reviewed Asia subnet shape/ + ) +}) + +test('rejects incomplete NAT, backend, and URL routing shapes', () => { + const nat = structuredClone(resources) + nat[2].change.after.subnetwork[0].source_ip_ranges_to_nat = [] + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: nat }, config), + /reviewed Asia NAT shape/ + ) + + const backend = structuredClone(resources) + backend[5].change.after.backend[0].group = 'projects/p/zones/asia-east2-a/instanceGroups/wrong' + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: backend }, config), + /reviewed Asia backend shape/ + ) + + const route = structuredClone(resources) + route[6].change.after.path_matcher[0].default_service = + 'projects/p/global/backendServices/wrong' + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: route }, config), + /no exact backend route/ + ) +}) diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.mjs new file mode 100644 index 00000000000..7307295d206 --- /dev/null +++ b/cloud/dev/scripts/validate-relay-capacity-plan.mjs @@ -0,0 +1,519 @@ +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +const SERVICE_ACCOUNT_EMAIL = + /^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com$/ + +function parseArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of ['mode', 'cell-id', 'hard-cap', 'unobserved-bound']) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (!['cell', 'bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(values.mode)) { + throw new Error('--mode must be cell, bootstrap-cell, same-cap-cell, or same-cap-image') + } + const integer = (key) => { + const value = Number(values[key]) + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`--${key} is invalid`) + return value + } + if (!values.image) throw new Error('missing --image') + if (values.mode === 'bootstrap-cell' && !values['capacity-service-account']) { + throw new Error('missing --capacity-service-account') + } + if ( + values.mode === 'same-cap-cell' && + (!values['rollback-image'] || + !values['rehome-director-service-account'] || + !values['rehome-audience']) + ) throw new Error('same-cap validation requires rollback image and rehome trust config') + if (values.mode === 'same-cap-image' && !values['rollback-image']) { + throw new Error('same-cap image validation requires a rollback image') + } + if ( + values['capacity-service-account'] !== undefined && + !SERVICE_ACCOUNT_EMAIL.test(values['capacity-service-account']) + ) { + throw new Error('--capacity-service-account is invalid') + } + return { + mode: values.mode, + cellId: values['cell-id'], + hardCap: integer('hard-cap'), + unobservedBound: integer('unobserved-bound'), + image: values.image, + capacityServiceAccount: values['capacity-service-account'], + rollbackImage: values['rollback-image'], + rehomeDirectorServiceAccount: values['rehome-director-service-account'], + rehomeAudience: values['rehome-audience'] + } +} + +function mutations(plan) { + if (!Array.isArray(plan.resource_changes)) throw new Error('Terraform plan has no resource changes') + return plan.resource_changes.filter(({ change }) => { + const actions = change?.actions + return Array.isArray(actions) && !actions.every((action) => ['no-op', 'read'].includes(action)) + }) +} + +function sameActions(change, expected) { + return JSON.stringify(change.change?.actions) === JSON.stringify(expected) +} + +function changedPaths(before, after, path = []) { + if (Object.is(before, after)) return [] + const beforeObject = before !== null && typeof before === 'object' + const afterObject = after !== null && typeof after === 'object' + if (!beforeObject || !afterObject || Array.isArray(before) !== Array.isArray(after)) { + return [path.join('.')] + } + const keys = new Set([...Object.keys(before), ...Object.keys(after)]) + return [...keys].flatMap((key) => changedPaths(before[key], after[key], [...path, key])) +} + +function unknownPaths(value, path = []) { + if (value === true) return [path.join('.')] + if (value === null || typeof value !== 'object') return [] + return Object.entries(value).flatMap(([key, nested]) => unknownPaths(nested, [...path, key])) +} + +function valueAtPath(value, path) { + return path.split('.').reduce((current, key) => current?.[key], value) +} + +function providerDefaultPaths(change, paths) { + const empty = (value) => + value === '' || + value === 0 || + (Array.isArray(value) && value.length === 0) || + (value !== null && + typeof value === 'object' && + !Array.isArray(value) && + Object.keys(value).length === 0) + return paths.filter( + (path) => + empty(valueAtPath(change.change.before, path)) && + valueAtPath(change.change.after, path) === null + ) +} + +function canonicalResourcePaths(change, paths) { + const canonical = (value) => + typeof value === 'string' + ? value.replace('https://www.googleapis.com/compute/v1/', '') + : value + return paths.filter( + (path) => + canonical(valueAtPath(change.change.before, path)) === + canonical(valueAtPath(change.change.after, path)) + ) +} + +function bootstrapRestartNormalizationPaths(change, mode) { + if (!['bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(mode)) return [] + const policyMatches = + valueAtPath(change.change.before, 'update_policy.0.minimal_action') === 'RESTART' && + valueAtPath(change.change.after, 'update_policy.0.minimal_action') === 'REPLACE' + const priorVersion = valueAtPath(change.change.before, 'version.0.name') + const versionMatches = + typeof priorVersion === 'string' && + /^0\/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}\+00:00$/.test(priorVersion) && + valueAtPath(change.change.after, 'version.0.name') === 'primary' + return policyMatches && versionMatches + ? ['update_policy.0.minimal_action', 'version.0.name'] + : [] +} + +function requireOnlyPaths(change, allowed, required = [], allowedUnknown = new Set()) { + const paths = changedPaths(change.change.before, change.change.after) + const unknown = unknownPaths(change.change.after_unknown) + const unexpected = paths.filter((path) => !allowed.has(path)) + const unexpectedUnknown = unknown.filter((path) => !allowedUnknown.has(path)) + if ( + unexpected.length > 0 || + unexpectedUnknown.length > 0 || + required.some((path) => !paths.includes(path)) + ) { + throw new Error(`${change.address} changes outside the reviewed capacity fields`) + } +} + +function relayImage(script) { + const lines = script.split('\n') + const starts = lines.flatMap((line, index) => + line === 'docker run --detach \\' ? [index] : []) + const commands = starts.map((start) => { + const end = lines.findIndex((line, index) => index > start && !line.endsWith(' \\')) + return end < 0 ? [] : lines.slice(start, end + 1) + }) + const relayCommands = commands.filter((command) => + command.filter((line) => line === ' --name orca-relay \\').length === 1) + if (relayCommands.length !== 1) return null + const command = relayCommands[0] + const image = /^ '([^'\n]+@sha256:[a-f0-9]{64})'$/.exec(command.at(-1))?.[1] + const digests = [...command.join('\n').matchAll(/'([^'\n]+@sha256:[a-f0-9]{64})'/g)] + return image && digests.length === 1 ? image : null +} + +function normalizedStartupScript( + script, + stripCapacityIdentity = false, + stripRehomeConfig = false, + preserveCapacity = false +) { + const image = relayImage(script) + if (!image) throw new Error('cell plan startup script has no Relay image') + const digest = image.split('@')[1] + const capacityAssignment = + /^ printf 'ORCA_RELAY_CELL_CONNECTION_(?:HARD_CAP|UNOBSERVED_BOUND)=%s\\n' '[0-9]+'$/ + const capacityIdentity = + /^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/ + const rehomeConfig = + /^ printf 'ORCA_RELAY_REHOME_(?:DIRECTOR_SERVICE_ACCOUNT|AUDIENCE)=%s\\n' '[^'\n]+'$/ + return script + .split('\n') + .filter( + (line) => + (preserveCapacity || !capacityAssignment.test(line)) && + (!stripCapacityIdentity || !capacityIdentity.test(line)) && + (!stripRehomeConfig || !rehomeConfig.test(line)) + ) + .join('\n') + .replaceAll(image, '') + .replaceAll(digest, '') +} + +function hasExactSingleAssignment(lines, pattern, expected) { + const assignments = lines.filter((line) => pattern.test(line)) + return assignments.length === 1 && assignments[0] === expected +} + +function requireDesiredStartupScript(script, config) { + const lines = typeof script === 'string' ? script.split('\n') : [] + const expected = [ + [ + /^ printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '[0-9]+'$/, + ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${config.hardCap}'` + ], + [ + /^ printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '[0-9]+'$/, + ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'` + ] + ] + if (config.mode === 'bootstrap-cell') { + expected.push([ + /^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/, + ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'` + ]) + } + if (config.mode === 'same-cap-cell') { + expected.push( + [ + /^ printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '[^'\n]+'$/, + ` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${config.rehomeDirectorServiceAccount}'` + ], + [ + /^ printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '[^'\n]+'$/, + ` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${config.rehomeAudience}'` + ] + ) + } + if ( + typeof script !== 'string' || + relayImage(script) !== config.image || + expected.some(([pattern, line]) => !hasExactSingleAssignment(lines, pattern, line)) + ) { + throw new Error('cell plan does not contain the reviewed image and capacity') + } +} + +function plannedResources(module) { + if (!module) return [] + return [ + ...(module.resources ?? []), + ...(module.child_modules ?? []).flatMap(plannedResources) + ] +} + +function canonicalResource(value) { + return typeof value === 'string' + ? value.replace('https://www.googleapis.com/compute/v1/', '') + : value +} + +function requireDesiredPlannedCell(plan, config) { + const resources = plannedResources(plan.planned_values?.root_module) + const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` + const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]` + const template = resources.find(({ address }) => address === templateAddress)?.values + const manager = resources.find(({ address }) => address === managerAddress)?.values + if (!template || !manager) throw new Error('convergence plan has no exact planned C26 state') + requireDesiredStartupScript(template.metadata_startup_script, config) + const templateReference = canonicalResource(template.self_link ?? template.id) + const managerReference = canonicalResource(manager.version?.[0]?.instance_template) + if (!templateReference || templateReference !== managerReference) { + throw new Error('convergence plan does not bind the MIG to the reviewed template') + } +} + +function validateManagerUpdate(manager, config) { + if (!sameActions(manager, ['update'])) { + throw new Error('cell plan has unexpected MIG actions') + } + const managerComputed = new Set([ + 'fingerprint', + 'operation', + 'status', + 'version.0.instance_template' + ]) + const managerUnknown = unknownPaths(manager.change.after_unknown) + requireOnlyPaths( + manager, + new Set([ + 'version.0.instance_template', + ...bootstrapRestartNormalizationPaths(manager, config.mode), + ...managerUnknown.filter((path) => managerComputed.has(path)) + ]), + ['version.0.instance_template'], + managerComputed + ) +} + +function requireReplacementTemplateDependency(plan, template, manager) { + const configuredManager = plan.configuration?.root_module?.resources?.find( + ({ address }) => address === 'google_compute_instance_group_manager.relay_gce_cell' + ) + const expectedVersionExpression = [{ + instance_template: { + references: ['google_compute_instance_template.relay_gce_cell', 'each.key'] + }, + name: { constant_value: 'primary' } + }] + if ( + JSON.stringify(configuredManager?.expressions?.version) !== + JSON.stringify(expectedVersionExpression) || + template.change.after?.self_link != null || + template.change.after_unknown?.self_link !== true || + manager.change.after?.version?.[0]?.instance_template != null || + manager.change.after_unknown?.version?.[0]?.instance_template !== true + ) { + throw new Error('cell plan does not bind the MIG to the reviewed template dependency') + } +} + +function cellPlan(plan, changes, config) { + const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` + const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]` + const template = changes.find( + ({ address, deposed }) => address === templateAddress && deposed === undefined + ) + const manager = changes.find(({ address }) => address === managerAddress) + const obsoleteTemplates = changes.filter( + ({ address, deposed }) => address === templateAddress && typeof deposed === 'string' + ) + const allowsObsoleteTemplates = + config.mode === 'same-cap-image' && + obsoleteTemplates.length > 0 && + obsoleteTemplates.every((change) => sameActions(change, ['delete'])) + if ( + !template || + !manager || + changes.length !== 2 + obsoleteTemplates.length || + (obsoleteTemplates.length > 0 && !allowsObsoleteTemplates) + ) { + throw new Error('cell plan must change only the exact instance template and MIG') + } + if (!sameActions(template, ['create', 'delete']) || !sameActions(manager, ['update'])) { + throw new Error('cell plan has unexpected replacement actions') + } + const templateComputed = new Set([ + 'confidential_instance_config', + 'creation_timestamp', + 'disk.0.architecture', + 'disk.0.interface', + 'disk.0.mode', + 'disk.0.provisioned_iops', + 'disk.0.provisioned_throughput', + 'disk.0.type', + 'id', + 'metadata_fingerprint', + 'name', + 'network_interface.0.internal_ipv6_prefix_length', + 'network_interface.0.ipv6_access_type', + 'network_interface.0.ipv6_address', + 'network_interface.0.name', + 'network_interface.0.network', + 'network_interface.0.stack_type', + 'network_interface.0.subnetwork_project', + 'numeric_id', + 'region', + 'self_link', + 'self_link_unique', + 'tags_fingerprint' + ]) + const templateDefaults = [ + 'description', + 'disk.0.disk_name', + 'disk.0.guest_os_features', + 'disk.0.labels', + 'disk.0.resource_manager_tags', + 'disk.0.resource_policies', + 'disk.0.source', + 'disk.0.source_snapshot', + 'instance_description', + 'key_revocation_action_type', + 'min_cpu_platform', + 'network_interface.0.network_ip', + 'network_interface.0.nic_type', + 'network_interface.0.queue_count', + 'scheduling.0.availability_domain', + 'scheduling.0.instance_termination_action', + 'scheduling.0.min_node_cpus', + 'scheduling.0.termination_time' + ] + const templateCanonicalResources = [ + 'disk.0.source_image', + 'network_interface.0.subnetwork' + ] + const templateUnknown = unknownPaths(template.change.after_unknown) + requireOnlyPaths( + template, + new Set([ + 'metadata_startup_script', + ...templateUnknown.filter((path) => templateComputed.has(path)), + ...providerDefaultPaths(template, templateDefaults), + ...canonicalResourcePaths(template, templateCanonicalResources) + ]), + ['metadata_startup_script'], + templateComputed + ) + validateManagerUpdate(manager, config) + requireReplacementTemplateDependency(plan, template, manager) + const beforeScript = template.change.before?.metadata_startup_script + const script = template.change.after?.metadata_startup_script + requireDesiredStartupScript(script, config) + const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode) + if ( + typeof beforeScript !== 'string' || + (sameCap && relayImage(beforeScript) !== config.rollbackImage) || + normalizedStartupScript( + beforeScript, + config.mode === 'bootstrap-cell', + config.mode === 'same-cap-cell', + sameCap + ) !== normalizedStartupScript( + script, + config.mode === 'bootstrap-cell', + config.mode === 'same-cap-cell', + sameCap + ) + ) { + throw new Error('cell plan does not contain the reviewed image and capacity') + } +} + +function convergenceCellPlan(plan, changes, config) { + const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` + const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]` + const manager = changes.find(({ address }) => address === managerAddress) + const obsoleteTemplates = changes.filter( + ({ address, deposed }) => address === templateAddress && typeof deposed === 'string' + ) + if ( + (!manager && obsoleteTemplates.length === 0) || + (obsoleteTemplates.length > 1 && config.mode !== 'same-cap-image') || + changes.length !== (manager ? 1 : 0) + obsoleteTemplates.length + ) { + throw new Error('cell convergence plan changes outside the exact template and MIG') + } + if (manager) validateManagerUpdate(manager, config) + if (obsoleteTemplates.some((change) => !sameActions(change, ['delete']))) { + throw new Error('cell convergence plan has unexpected obsolete-template actions') + } + requireDesiredPlannedCell(plan, config) +} + +export function validateCapacityPlan(plan, config) { + if (!['cell', 'bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(config.mode)) { + throw new Error('capacity Terraform plans may change only a cell') + } + if ( + config.mode === 'bootstrap-cell' && + !SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '') + ) { + throw new Error('capacity Terraform plan has an invalid service account') + } + if ( + config.mode === 'same-cap-cell' && + (!SERVICE_ACCOUNT_EMAIL.test(config.rehomeDirectorServiceAccount ?? '') || + !/^https:\/\/[^/]+\/v1\/admin\/host-drain$/.test(config.rehomeAudience ?? '') || + !/^.+@sha256:[a-f0-9]{64}$/.test(config.rollbackImage ?? '')) + ) throw new Error('same-cap Terraform plan has invalid rehome trust config') + if ( + config.mode === 'same-cap-image' && + !/^.+@sha256:[a-f0-9]{64}$/.test(config.rollbackImage ?? '') + ) throw new Error('same-cap image Terraform plan has an invalid rollback image') + const changes = mutations(plan) + if (changes.length === 0) { + return { + mode: config.mode, + changes: 0, + ...(config.mode === 'same-cap-image' ? { changeKind: 'none' } : {}) + } + } + const replacement = changes.some( + ({ address, deposed, change }) => + address === `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` && + deposed === undefined && + JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']) + ) + if (replacement) cellPlan(plan, changes, config) + else convergenceCellPlan(plan, changes, config) + const obsoleteTemplateOnly = changes.every( + ({ address, deposed, change }) => + address === `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` && + typeof deposed === 'string' && + JSON.stringify(change?.actions) === JSON.stringify(['delete']) + ) + return { + mode: config.mode, + changes: changes.length, + ...(config.mode === 'same-cap-image' + ? { + changeKind: replacement + ? changes.some( + ({ address, deposed }) => + address === `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` && + typeof deposed === 'string' + ) + ? 'replacement-with-obsolete-template' + : 'replacement' + : obsoleteTemplateOnly + ? 'obsolete-template-delete' + : 'manager-convergence' + } + : {}) + } +} + +export function main(argv = process.argv.slice(2)) { + const config = parseArguments(argv) + const plan = JSON.parse(readFileSync(0, 'utf8')) + process.stdout.write(`${JSON.stringify({ event: 'relay_capacity_plan_verified', ...validateCapacityPlan(plan, config) })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + main() + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } +} diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs new file mode 100644 index 00000000000..207285dc570 --- /dev/null +++ b/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs @@ -0,0 +1,646 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { validateCapacityPlan as validateCapacityPlanRaw } from './validate-relay-capacity-plan.mjs' + +const config = { + cellId: 'staging-gce-c3', + hardCap: 1_000, + unobservedBound: 60 +} + +function replacementConfiguration(references = [ + 'google_compute_instance_template.relay_gce_cell', + 'each.key' +]) { + return { + root_module: { + resources: [{ + address: 'google_compute_instance_group_manager.relay_gce_cell', + expressions: { + version: [{ + instance_template: { references }, + name: { constant_value: 'primary' } + }] + } + }] + } + } +} + +function validateCapacityPlan(plan, planConfig) { + const replacement = plan.resource_changes.some(({ change }) => + JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete'])) + return validateCapacityPlanRaw( + replacement && !plan.configuration + ? { ...plan, configuration: replacementConfiguration() } + : plan, + planConfig + ) +} + +test('accepts only the exact canary template replacement and MIG update', () => { + const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}` + const startupScript = (cap, bound, selectedImage, extra = '') => + [ + ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`, + ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`, + `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, + `docker pull '${selectedImage}'`, + extra, + 'docker run --detach \\', + ' --name cloud-sql-proxy \\', + ` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`, + 'docker run --detach \\', + ' --name orca-relay \\', + ` '${selectedImage}'` + ].join('\n') + const script = startupScript(1_000, 60, image) + const template = { + address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]', + change: { + actions: ['create', 'delete'], + before: { + metadata_startup_script: startupScript( + 600, + 60, + `us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}` + ) + }, + after: { metadata_startup_script: script, self_link: null }, + after_unknown: { self_link: true } + } + } + const manager = { + address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]', + change: { + actions: ['update'], + before: { target_size: 1, version: [{ instance_template: 'old' }] }, + after: { target_size: 1, version: [{ instance_template: null }] }, + after_unknown: { version: [{ instance_template: true }] } + } + } + const cellConfig = { ...config, mode: 'cell', image } + assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), { + mode: 'cell', + changes: 0 + }) + assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), { + mode: 'cell', + changes: 2 + }) + const wrongTemplateManager = structuredClone(manager) + wrongTemplateManager.change.after.version[0].instance_template = + 'projects/project/global/instanceTemplates/unreviewed' + assert.throws( + () => validateCapacityPlan( + { resource_changes: [template, wrongTemplateManager] }, + cellConfig + ), + /does not bind the MIG/ + ) + assert.throws( + () => validateCapacityPlanRaw( + { + resource_changes: [template, manager], + configuration: replacementConfiguration([ + 'google_compute_instance_template.relay_gce_cell', + 'var.unreviewed_key' + ]) + }, + cellConfig + ), + /reviewed template dependency/ + ) + assert.throws( + () => validateCapacityPlanRaw( + { resource_changes: [template, manager] }, + cellConfig + ), + /reviewed template dependency/ + ) + const capacityServiceAccount = + 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com' + const bootstrapTemplate = structuredClone(template) + const bootstrapManager = structuredClone(manager) + bootstrapTemplate.change.after.metadata_startup_script = [ + ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`, + script + ].join('\n') + const bootstrapConfig = { + ...cellConfig, + mode: 'bootstrap-cell', + capacityServiceAccount + } + const plannedValues = (startupScript) => ({ + root_module: { + resources: [ + { + address: bootstrapTemplate.address, + values: { + metadata_startup_script: startupScript, + self_link: 'projects/project/global/instanceTemplates/c26-reviewed' + } + }, + { + address: bootstrapManager.address, + values: { + version: [{ + instance_template: + 'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed' + }] + } + } + ] + } + }) + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, bootstrapManager] }, + bootstrapConfig + ), + { mode: 'bootstrap-cell', changes: 2 } + ) + const managerOnly = structuredClone(bootstrapManager) + assert.deepEqual( + validateCapacityPlan( + { + resource_changes: [managerOnly], + planned_values: plannedValues( + bootstrapTemplate.change.after.metadata_startup_script + ) + }, + bootstrapConfig + ), + { mode: 'bootstrap-cell', changes: 1 } + ) + const decoyPlannedValues = plannedValues( + bootstrapTemplate.change.after.metadata_startup_script.replaceAll( + image, + `us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}` + ) + `\n# decoy '${image}'` + ) + assert.throws( + () => validateCapacityPlan( + { resource_changes: [managerOnly], planned_values: decoyPlannedValues }, + bootstrapConfig + ), + /reviewed image and capacity/ + ) + const obsoleteTemplate = structuredClone(bootstrapTemplate) + obsoleteTemplate.deposed = 'retired-template' + obsoleteTemplate.change.actions = ['delete'] + obsoleteTemplate.change.after = null + assert.deepEqual( + validateCapacityPlan( + { + resource_changes: [managerOnly, obsoleteTemplate], + planned_values: plannedValues( + bootstrapTemplate.change.after.metadata_startup_script + ) + }, + bootstrapConfig + ), + { mode: 'bootstrap-cell', changes: 2 } + ) + assert.deepEqual( + validateCapacityPlan( + { + resource_changes: [obsoleteTemplate], + planned_values: plannedValues( + bootstrapTemplate.change.after.metadata_startup_script + ) + }, + bootstrapConfig + ), + { mode: 'bootstrap-cell', changes: 1 } + ) + const wrongManagerReference = plannedValues( + bootstrapTemplate.change.after.metadata_startup_script + ) + wrongManagerReference.root_module.resources[1].values.version[0].instance_template = + 'projects/project/global/instanceTemplates/not-reviewed' + assert.throws( + () => validateCapacityPlan( + { resource_changes: [managerOnly], planned_values: wrongManagerReference }, + bootstrapConfig + ), + /does not bind the MIG/ + ) + assert.throws( + () => validateCapacityPlan( + { resource_changes: [managerOnly] }, + bootstrapConfig + ), + /no exact planned C26 state/ + ) + const restartedBootstrapManager = structuredClone(bootstrapManager) + restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }] + restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }] + restartedBootstrapManager.change.before.version[0].name = + '0/2026-08-10 23:30:14.196895+00:00' + restartedBootstrapManager.change.after.version[0].name = 'primary' + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, restartedBootstrapManager] }, + bootstrapConfig + ), + { mode: 'bootstrap-cell', changes: 2 } + ) + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [template, restartedBootstrapManager] }, + cellConfig + ), + /outside the reviewed capacity fields/ + ) + const unrecognizedRestartManager = structuredClone(restartedBootstrapManager) + unrecognizedRestartManager.change.before.version[0].name = 'operator-version' + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, unrecognizedRestartManager] }, + bootstrapConfig + ), + /outside the reviewed capacity fields/ + ) + const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager) + unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH' + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] }, + bootstrapConfig + ), + /outside the reviewed capacity fields/ + ) + const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager) + unrecognizedPrimaryVersion.change.after.version[0].name = 'other' + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] }, + bootstrapConfig + ), + /outside the reviewed capacity fields/ + ) + const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager) + unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH' + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] }, + bootstrapConfig + ), + /outside the reviewed capacity fields/ + ) + const missingRestartPolicy = structuredClone(restartedBootstrapManager) + delete missingRestartPolicy.change.before.update_policy + delete missingRestartPolicy.change.after.update_policy + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, missingRestartPolicy] }, + bootstrapConfig + ), + /outside the reviewed capacity fields/ + ) + const missingRestartVersion = structuredClone(restartedBootstrapManager) + missingRestartVersion.change.before.version[0].name = 'primary' + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, missingRestartVersion] }, + bootstrapConfig + ), + /outside the reviewed capacity fields/ + ) + const duplicateHardCapTemplate = structuredClone(template) + duplicateHardCapTemplate.change.after.metadata_startup_script = [ + script, + ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'` + ].join('\n') + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] }, + cellConfig + ), + /reviewed image and capacity/ + ) + const duplicateIdentityTemplate = structuredClone(bootstrapTemplate) + duplicateIdentityTemplate.change.after.metadata_startup_script = [ + duplicateIdentityTemplate.change.after.metadata_startup_script, + ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'` + ].join('\n') + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] }, + bootstrapConfig + ), + /reviewed image and capacity/ + ) + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, bootstrapManager] }, + { ...bootstrapConfig, capacityServiceAccount: 'invalid' } + ), + /invalid service account/ + ) + assert.throws( + () => + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, bootstrapManager] }, + cellConfig + ), + /reviewed image and capacity/ + ) + manager.change.after.target_size = 0 + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), + /outside the reviewed capacity fields/ + ) + manager.change.after.target_size = 1 + template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad') + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), + /reviewed image and capacity/ + ) + template.change.after.metadata_startup_script = startupScript( + 1_000, + 60, + image, + 'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP=' + ) + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), + /reviewed image and capacity/ + ) + template.change.after.metadata_startup_script = script + template.change.after_unknown = { + id: true, + self_link: true, + disk: [{ architecture: true }] + } + manager.change.after_unknown = { + fingerprint: true, + version: [{ instance_template: true }] + } + assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), { + mode: 'cell', + changes: 2 + }) + template.change.before.description = '' + template.change.after.description = null + template.change.before.disk = [{ + architecture: '', + source_image: 'projects/cos-cloud/global/images/cos-stable-1' + }] + template.change.after.disk = [{ + source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1' + }] + template.change.after_unknown.disk = [{ architecture: true }] + assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), { + mode: 'cell', + changes: 2 + }) + template.change.after.disk[0].source_image = + 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different' + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), + /outside the reviewed capacity fields/ + ) + template.change.after.disk[0].source_image = + 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1' + manager.change.after_unknown = { target_size: true } + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), + /outside the reviewed capacity fields/ + ) +}) + +test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => { + const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` + const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` + const directorIdentity = 'relay-director@project.iam.gserviceaccount.com' + const audience = 'https://relay.example.com/v1/admin/host-drain' + const startup = ({ selectedImage, cap = 1_000, trust = false }) => [ + ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`, + ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, + ...(trust ? [ + ` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`, + ` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'` + ] : []), + `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, + `docker pull '${selectedImage}'`, + 'docker run --detach \\', + ' --name orca-relay \\', + ` '${selectedImage}'` + ].join('\n') + const template = { + address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]', + change: { + actions: ['create', 'delete'], + before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) }, + after: { + metadata_startup_script: startup({ selectedImage: image, trust: true }), + self_link: null + }, + after_unknown: { self_link: true } + } + } + const manager = { + address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]', + change: { + actions: ['update'], + before: { target_size: 1, version: [{ instance_template: 'old' }] }, + after: { target_size: 1, version: [{ instance_template: null }] }, + after_unknown: { version: [{ instance_template: true }] } + } + } + const sameCapConfig = { + ...config, + mode: 'same-cap-cell', + image, + rollbackImage, + rehomeDirectorServiceAccount: directorIdentity, + rehomeAudience: audience + } + assert.deepEqual( + validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig), + { mode: 'same-cap-cell', changes: 2 } + ) + // A pre-template-apply rollback resume validates drift for the image the + // cell already serves: the template leaves and re-enters the rollback image. + const resumeTemplate = structuredClone(template) + resumeTemplate.change.after.metadata_startup_script = startup({ + selectedImage: rollbackImage, + trust: true + }) + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [resumeTemplate, manager] }, + { ...sameCapConfig, image: rollbackImage } + ), + { mode: 'same-cap-cell', changes: 2 } + ) + const asiaTemplate = structuredClone(template) + const asiaManager = structuredClone(manager) + asiaTemplate.address = + 'google_compute_instance_template.relay_gce_cell["production-gce-c28"]' + asiaManager.address = + 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]' + asiaTemplate.change.before.metadata_startup_script = startup({ + selectedImage: rollbackImage, + cap: 3_000 + }) + asiaTemplate.change.after.metadata_startup_script = startup({ + selectedImage: image, + cap: 3_000, + trust: true + }) + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [asiaTemplate, asiaManager] }, + { ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 } + ), + { mode: 'same-cap-cell', changes: 2 } + ) + const changedCap = structuredClone(template) + changedCap.change.before.metadata_startup_script = startup({ + selectedImage: rollbackImage, + cap: 600 + }) + assert.throws( + () => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig), + /reviewed image and capacity/ + ) + assert.throws( + () => validateCapacityPlan( + { resource_changes: [template, manager] }, + { ...sameCapConfig, rollbackImage: image } + ), + /reviewed image and capacity/ + ) + const wrongTrust = structuredClone(template) + wrongTrust.change.after.metadata_startup_script = startup({ + selectedImage: image, + trust: true + }).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com') + assert.throws( + () => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig), + /reviewed image and capacity/ + ) + + const imageOnly = structuredClone(template) + imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image }) + const imageOnlyConfig = { + ...config, + mode: 'same-cap-image', + image, + rollbackImage + } + assert.deepEqual( + validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig), + { mode: 'same-cap-image', changes: 2, changeKind: 'replacement' } + ) + assert.throws( + () => validateCapacityPlan( + { resource_changes: [imageOnly, manager] }, + { ...imageOnlyConfig, rollbackImage: image } + ), + /reviewed image and capacity/ + ) + const changedTrust = structuredClone(imageOnly) + changedTrust.change.after.metadata_startup_script += + `\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'` + assert.throws( + () => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig), + /reviewed image and capacity/ + ) + const plannedValues = { + root_module: { + resources: [ + { + address: imageOnly.address, + values: { + metadata_startup_script: imageOnly.change.after.metadata_startup_script, + self_link: 'projects/project/global/instanceTemplates/target' + } + }, + { + address: manager.address, + values: { + version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }] + } + } + ] + } + } + const obsoleteTemplate = structuredClone(imageOnly) + obsoleteTemplate.deposed = 'obsolete' + obsoleteTemplate.change.actions = ['delete'] + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [obsoleteTemplate], planned_values: plannedValues }, + imageOnlyConfig + ), + { mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' } + ) + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [imageOnly, manager, obsoleteTemplate] }, + imageOnlyConfig + ), + { mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' } + ) + const anotherObsoleteTemplate = { + ...structuredClone(obsoleteTemplate), + deposed: 'another-obsolete' + } + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] }, + imageOnlyConfig + ), + { mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' } + ) + assert.deepEqual( + validateCapacityPlan( + { + resource_changes: [obsoleteTemplate, anotherObsoleteTemplate], + planned_values: plannedValues + }, + imageOnlyConfig + ), + { mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' } + ) + assert.deepEqual( + validateCapacityPlan( + { + resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate], + planned_values: plannedValues + }, + imageOnlyConfig + ), + { mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' } + ) + const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate) + invalidObsoleteTemplate.change.actions = ['update'] + assert.throws( + () => validateCapacityPlan( + { resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] }, + imageOnlyConfig + ), + /change only the exact instance template and MIG/ + ) + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [manager], planned_values: plannedValues }, + imageOnlyConfig + ), + { mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' } + ) +}) diff --git a/cloud/dev/scripts/verify-relay-capacity-transition.mjs b/cloud/dev/scripts/verify-relay-capacity-transition.mjs new file mode 100644 index 00000000000..e5ebe77d45f --- /dev/null +++ b/cloud/dev/scripts/verify-relay-capacity-transition.mjs @@ -0,0 +1,473 @@ +import { pathToFileURL } from 'node:url' + +const CAPACITY_PROTOCOL = 2 + +function integer(value, name) { + const parsed = Number(value) + if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`) + return parsed +} + +function signedInteger(value, name) { + if (typeof value !== 'number' || !Number.isSafeInteger(value)) { + throw new Error(`${name} is invalid`) + } + return value +} + +export function parseCapacityTransitionArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of [ + 'director-origin', + 'cell-origin', + 'cell-id', + 'heartbeat', + 'admission', + 'draining', + 'activity' + ]) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (!['fresh', 'stale', 'either'].includes(values.heartbeat)) { + throw new Error('--heartbeat must be fresh, stale, or either') + } + if ( + !['general', 'migration-only', 'general-or-migration-only', 'non-general', 'either'].includes( + values.admission + ) + ) { + throw new Error( + '--admission must be general, migration-only, general-or-migration-only, non-general, or either' + ) + } + if (!['required', 'forbidden', 'either'].includes(values.draining)) { + throw new Error('--draining must be required, forbidden, or either') + } + if (!['quiescent', 'restart-safe', 'allowed'].includes(values.activity)) { + throw new Error('--activity must be quiescent, restart-safe, or allowed') + } + const runtime = values.runtime ?? 'required' + if (!['required', 'unavailable'].includes(runtime)) { + throw new Error('--runtime must be required or unavailable') + } + if ( + values.activity === 'restart-safe' && + runtime === 'required' && + (values.admission !== 'migration-only' || values.draining !== 'required') + ) { + throw new Error('restart-safe activity requires migration-only admission and draining') + } + if ( + runtime === 'unavailable' && + (values.heartbeat !== 'stale' || + values.admission !== 'migration-only' || + values.draining !== 'either' || + values.activity !== 'restart-safe') + ) { + throw new Error('unavailable runtime requires stale migration-only durable state') + } + const origin = new URL(values['director-origin']) + const cellOrigin = new URL(values['cell-origin']) + if ( + origin.protocol !== 'https:' || + origin.origin !== values['director-origin'] || + cellOrigin.protocol !== 'https:' || + cellOrigin.origin !== values['cell-origin'] + ) { + throw new Error('origins must be canonical HTTPS origins') + } + const hardCap = values['hard-cap'] === undefined + ? undefined + : integer(values['hard-cap'], '--hard-cap') + const unobservedBound = values['unobserved-bound'] === undefined + ? undefined + : integer(values['unobserved-bound'], '--unobserved-bound') + if ((hardCap === undefined) !== (unobservedBound === undefined)) { + throw new Error('capacity expectations must be paired') + } + if (runtime === 'unavailable' && hardCap !== undefined) { + throw new Error('unavailable runtime cannot prove live capacity') + } + const expectedImageDigests = values['expected-image-digests']?.split(',') ?? [] + if ( + new Set(expectedImageDigests).size !== expectedImageDigests.length || + expectedImageDigests.some((digest) => !/^sha256:[a-f0-9]{64}$/.test(digest)) + ) { + throw new Error('--expected-image-digests is invalid') + } + if (runtime === 'unavailable' && expectedImageDigests.length > 0) { + throw new Error('unavailable runtime cannot prove a live image') + } + const regionalRehomeProtocol = values['regional-rehome-protocol'] === undefined + ? undefined + : integer(values['regional-rehome-protocol'], '--regional-rehome-protocol') + if (regionalRehomeProtocol !== undefined && ![0, 1].includes(regionalRehomeProtocol)) { + throw new Error('--regional-rehome-protocol must be 0 or 1') + } + if (runtime === 'unavailable' && regionalRehomeProtocol !== undefined) { + throw new Error('unavailable runtime cannot prove the regional rehome protocol') + } + return { + directorOrigin: origin.origin, + cellOrigin: cellOrigin.origin, + cellId: values['cell-id'], + heartbeat: values.heartbeat, + admission: values.admission, + draining: values.draining, + activity: values.activity, + runtime, + expectedImageDigests, + ...(regionalRehomeProtocol === undefined ? {} : { regionalRehomeProtocol }), + hardCap, + unobservedBound, + timeoutMs: integer(values['timeout-ms'] ?? 180_000, '--timeout-ms') + } +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({})) + if (!response.ok) throw new Error(`${label} returned ${response.status}`) + return body +} + +async function cellRuntime(fetchImpl, config, token) { + let response + try { + response = await fetchImpl(`${config.cellOrigin}/v1/admin/runtime-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1 }), + signal: AbortSignal.timeout(30_000) + }) + } catch (error) { + if (config.runtime === 'unavailable') return null + throw error + } + if ([502, 503, 504].includes(response.status)) { + await response.arrayBuffer().catch(() => undefined) + return null + } + return await responseJson(response, 'cell runtime status') +} + +function offlineRollbackMatches(status, config) { + if (status.admissionState !== 'migration-only') { + throw new Error('capacity transition admission does not match the required state') + } + const durableCounts = [ + status.activityLeases, + status.activityRequestUnits, + status.reservedRequests, + status.restartBlockingActivityLeases, + status.restartBlockingActivityRequestUnits, + status.outgoingMigrations, + status.incomingMigrations, + status.connectionCapacity?.pendingControlReservations + ] + const restartBlockingReservedRequests = signedInteger( + status.restartBlockingReservedRequests, + 'restart-blocking reserved requests' + ) + // Only a positive remainder is unexplained; the other gates reject real work. + return heartbeatMatches(status, config.heartbeat) && + durableCounts.every((value) => integer(value, 'durable activity count') === 0) && + restartBlockingReservedRequests <= 0 +} + +function directorActivityMatches(status, config, restartBlockingReservedRequests) { + const durable = [ + status.activityLeases, + status.reservedRequests, + status.outgoingMigrations, + status.incomingMigrations + ] + // Reconnect reservations survive replacement; draining prevents activation on this process. + const transient = [ + status.connectionCapacity?.observedConnections, + status.connectionCapacity?.inFlightConnections, + status.connectionCapacity?.reservedConnectionUnits, + status.connectionCapacity?.enforcedConnectionUnits, + status.connectionCapacity?.pendingControlReservations + ] + const restartSafe = config.activity !== 'restart-safe' || (() => { + // Only a positive remainder is unexplained; the other gates reject real work. + return integer( + status.restartBlockingActivityLeases, + 'restart-blocking activity leases' + ) === 0 && + integer( + status.restartBlockingActivityRequestUnits, + 'restart-blocking activity request units' + ) === 0 && + restartBlockingReservedRequests <= 0 && + integer(status.outgoingMigrations, 'outgoing migrations') === 0 && + integer(status.incomingMigrations, 'incoming migrations') === 0 + })() + const quiescent = [...durable, ...transient] + .filter((value) => value !== undefined) + .every((value) => integer(value, 'activity count') === 0) + if ( + (config.admission === 'general' && status.admissionState !== 'general') || + (config.admission === 'migration-only' && status.admissionState !== 'migration-only') || + // A failed same-cap canary leaves its cell migration-only; the documented + // rollback recovery must accept that state alongside a completed general roll. + (config.admission === 'general-or-migration-only' && + !['general', 'migration-only'].includes(status.admissionState)) || + (config.admission === 'non-general' && + !['existing-only', 'migration-only'].includes(status.admissionState)) + ) { + throw new Error('capacity transition admission does not match the required state') + } + return config.activity === 'allowed' || + (config.activity === 'restart-safe' ? restartSafe : quiescent) +} + +function capacityMatches(status, config) { + if (config.hardCap === undefined) return true + const capacity = status.connectionCapacity + return ( + capacity?.hardCap === config.hardCap && + capacity.unobservedBound === config.unobservedBound && + capacity.controlRebindReserve === 100 && + capacity.ordinaryConnectionLimit === config.hardCap - 100 && + capacity.normalAdmissionPause === config.hardCap - 100 - config.unobservedBound + ) +} + +function heartbeatMatches(status, expectation) { + if (expectation === 'either') return true + const fresh = status.connectionCapacity?.heartbeatFresh ?? status.runtime?.heartbeatFresh + return fresh === (expectation === 'fresh') +} + +function aggregateCount(value) { + const parsed = Number(value) + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null +} + +function signedAggregateCount(value) { + return typeof value === 'number' && Number.isSafeInteger(value) ? value : null +} + +function capacityObservation(capacity) { + if (capacity === null || capacity === undefined) return null + return { + hardCap: aggregateCount(capacity.hardCap), + controlRebindReserve: aggregateCount(capacity.controlRebindReserve), + ordinaryConnectionLimit: aggregateCount(capacity.ordinaryConnectionLimit), + unobservedBound: aggregateCount(capacity.unobservedBound), + normalAdmissionPause: aggregateCount(capacity.normalAdmissionPause), + observedConnections: aggregateCount(capacity.observedConnections), + inFlightConnections: aggregateCount(capacity.inFlightConnections), + reservedConnectionUnits: aggregateCount(capacity.reservedConnectionUnits), + enforcedConnectionUnits: aggregateCount(capacity.enforcedConnectionUnits), + pendingControlReservations: aggregateCount(capacity.pendingControlReservations), + heartbeatFresh: typeof capacity.heartbeatFresh === 'boolean' + ? capacity.heartbeatFresh + : null + } +} + +function transitionObservation(runtime, status) { + return { + runtimeAvailable: runtime !== null, + admissionState: ['general', 'migration-only', 'existing-only'].includes(status.admissionState) + ? status.admissionState + : null, + draining: typeof runtime?.draining === 'boolean' ? runtime.draining : null, + runtime: runtime === null + ? null + : { + totalConnections: aggregateCount(runtime.runtime?.totalConnections), + preAuthConnections: aggregateCount(runtime.runtime?.preAuthConnections), + inFlightConnections: aggregateCount(runtime.runtime?.inFlightConnections), + reservedConnectionUnits: aggregateCount(runtime.runtime?.reservedConnectionUnits), + enforcedConnectionUnits: aggregateCount(runtime.runtime?.enforcedConnectionUnits), + controls: aggregateCount(runtime.runtime?.controls), + splices: aggregateCount(runtime.runtime?.splices), + pendingSplices: aggregateCount(runtime.runtime?.pendingSplices), + queuedBytes: aggregateCount(runtime.runtime?.queuedBytes) + }, + director: { + activityLeases: aggregateCount(status.activityLeases), + activityRequestUnits: aggregateCount(status.activityRequestUnits), + reservedRequests: aggregateCount(status.reservedRequests), + restartBlockingActivityLeases: aggregateCount(status.restartBlockingActivityLeases), + restartBlockingActivityRequestUnits: + aggregateCount(status.restartBlockingActivityRequestUnits), + restartBlockingReservedRequests: + signedAggregateCount(status.restartBlockingReservedRequests), + outgoingMigrations: aggregateCount(status.outgoingMigrations), + incomingMigrations: aggregateCount(status.incomingMigrations) + }, + runtimeCapacity: capacityObservation(runtime?.connectionCapacity), + directorCapacity: capacityObservation(status.connectionCapacity), + runtimeHeartbeatFresh: typeof status.runtime?.heartbeatFresh === 'boolean' + ? status.runtime.heartbeatFresh + : null + } +} + +function runtimeQuiescent(runtime, config) { + if ( + runtime.role !== 'cell' || + runtime.cellId !== config.cellId || + runtime.cellUrl !== config.cellOrigin || + // Legacy pre-rehome images omit the field; the exact digest binds absence to protocol 0. + (config.regionalRehomeProtocol !== undefined && + (runtime.regionalRehomeProtocol ?? 0) !== config.regionalRehomeProtocol) || + (config.expectedImageDigests?.length > 0 && + !config.expectedImageDigests.includes(runtime.imageDigest)) + ) { + throw new Error('capacity transition runtime does not match the cell') + } + if ( + (config.draining === 'required' && runtime.draining !== true) || + (config.draining === 'forbidden' && runtime.draining === true) + ) { + return false + } + const counts = [runtime.runtime?.totalConnections, runtime.runtime?.preAuthConnections] + if (counts.some((value) => value === undefined)) { + throw new Error('capacity transition runtime is incomplete') + } + if (runtime.connectionCapacity !== null && runtime.connectionCapacity !== undefined) { + if (runtime.runtime?.enforcedConnectionUnits === undefined) { + throw new Error('capacity transition runtime is incomplete') + } + counts.push(runtime.runtime.enforcedConnectionUnits) + } + const quiescent = counts.every((value) => integer(value, 'runtime connection count') === 0) + const restartSafe = config.activity !== 'restart-safe' || + [ + runtime.runtime?.preAuthConnections, + runtime.runtime?.inFlightConnections, + runtime.runtime?.reservedConnectionUnits, + runtime.runtime?.controls, + runtime.runtime?.splices, + runtime.runtime?.pendingSplices, + runtime.runtime?.queuedBytes + ].every((value) => integer(value, 'live runtime count') === 0) + if ( + config.heartbeat === 'fresh' && + !capacityMatches({ connectionCapacity: runtime.connectionCapacity }, config) + ) { + return false + } + return config.activity === 'allowed' || + (config.activity === 'restart-safe' ? restartSafe : quiescent) +} + +export async function verifyCapacityTransition(config, overrides = {}) { + if ( + config.activity === 'restart-safe' && + config.runtime !== 'unavailable' && + (config.admission !== 'migration-only' || config.draining !== 'required') + ) { + throw new Error('restart-safe activity requires migration-only admission and draining') + } + const fetchImpl = overrides.fetch ?? fetch + const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) + const now = overrides.now ?? Date.now + const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN + if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') + const health = await responseJson( + await fetchImpl(`${config.directorOrigin}/health`, { + signal: AbortSignal.timeout(15_000) + }), + 'director health' + ) + if (health.ok !== true || health.connectionCapacityProtocol !== CAPACITY_PROTOCOL) { + throw new Error('director is not capacity-protocol compatible') + } + const deadline = now() + config.timeoutMs + let restartSafeSamples = 0 + let lastObservation = { runtimeAvailable: false } + for (;;) { + const runtime = await cellRuntime(fetchImpl, config, token) + lastObservation = { runtimeAvailable: runtime !== null } + if ((runtime === null) === (config.runtime === 'unavailable')) { + const result = await responseJson( + await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId: config.cellId }), + signal: AbortSignal.timeout(30_000) + }), + 'cell status' + ) + const status = result.status + if ( + status?.cellId !== config.cellId || + status.cellUrl !== config.cellOrigin || + status.runtime?.cellUrl !== config.cellOrigin + ) { + throw new Error('capacity transition director status does not match the cell') + } + lastObservation = transitionObservation(runtime, status) + const restartBlockingReservedRequests = + config.activity === 'restart-safe' + ? signedInteger( + status.restartBlockingReservedRequests, + 'restart-blocking reserved requests' + ) + : null + const matches = runtime === null + ? offlineRollbackMatches(status, config) + : runtimeQuiescent(runtime, config) && + directorActivityMatches(status, config, restartBlockingReservedRequests) && + capacityMatches(status, config) && + heartbeatMatches(status, config.heartbeat) + if (matches && (config.activity !== 'restart-safe' || restartSafeSamples === 1)) { + return { + cellId: status.cellId, + admissionState: status.admissionState, + assignments: integer(status.assignments, 'assignments'), + hardCap: runtime === null ? null : status.connectionCapacity?.hardCap ?? null, + unobservedBound: + runtime === null ? null : status.connectionCapacity?.unobservedBound ?? null, + heartbeatFresh: + status.connectionCapacity?.heartbeatFresh ?? status.runtime?.heartbeatFresh ?? false, + imageDigest: runtime?.imageDigest ?? null, + ...(config.activity === 'restart-safe' + ? { restartBlockingReservedRequests } + : {}) + } + } + restartSafeSamples = matches ? 1 : 0 + } else { + restartSafeSamples = 0 + } + if (config.activity === 'restart-safe') { + lastObservation = { + ...lastObservation, + restartSafeSamples, + requiredRestartSafeSamples: 2 + } + } + if (now() >= deadline) { + throw new Error( + `capacity transition verification timed out: ${JSON.stringify(lastObservation)}` + ) + } + await wait(5_000) + } +} + +export async function main(argv = process.argv.slice(2)) { + const result = await verifyCapacityTransition(parseCapacityTransitionArguments(argv)) + process.stdout.write(`${JSON.stringify({ event: 'relay_capacity_transition_verified', ...result })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs b/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs new file mode 100644 index 00000000000..fb865257c4a --- /dev/null +++ b/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs @@ -0,0 +1,1096 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + parseCapacityTransitionArguments, + verifyCapacityTransition +} from './verify-relay-capacity-transition.mjs' + +const imageDigest = `sha256:${'a'.repeat(64)}` +const config = { + directorOrigin: 'https://relay.example.com', + cellOrigin: 'https://c3.relay.example.com', + cellId: 'staging-gce-c3', + heartbeat: 'fresh', + admission: 'non-general', + draining: 'required', + activity: 'quiescent', + expectedImageDigests: [imageDigest], + hardCap: 1_000, + unobservedBound: 60, + timeoutMs: 1 +} + +function response(body) { + return Response.json(body) +} + +function harness({ + heartbeatFresh = true, + active = 0, + preAuthConnections = 0, + inFlightConnections = 0, + reservedConnectionUnits = 0, + enforcedConnectionUnits = active, + activityLeases = active, + activityRequestUnits = activityLeases, + reservedRequests = activityRequestUnits, + restartBlockingActivityLeases = activityLeases, + restartBlockingActivityRequestUnits = activityRequestUnits, + restartBlockingReservedRequests = reservedRequests, + outgoingMigrations = 0, + incomingMigrations = 0, + controls = 0, + splices = 0, + pendingSplices = 0, + queuedBytes = 0, + pendingControlReservations = 0, + runtimeHardCap = 1_000, + directorHardCap = 1_000, + runtimeImageDigest = imageDigest, + protocol = 2, + regionalRehomeProtocol = 1, + legacy = false, + admission = config.admission, + draining = config.draining +} = {}) { + return async (url) => { + const parsed = new URL(url) + if (parsed.pathname === '/health') { + return response({ ok: true, connectionCapacityProtocol: protocol }) + } + if (parsed.pathname === '/v1/admin/runtime-status') { + return response({ + role: 'cell', + cellId: config.cellId, + cellUrl: config.cellOrigin, + imageDigest: runtimeImageDigest, + ...(regionalRehomeProtocol === null ? {} : { regionalRehomeProtocol }), + draining: draining === 'required', + connectionCapacity: legacy + ? null + : { + hardCap: runtimeHardCap, + unobservedBound: 60, + controlRebindReserve: 100, + ordinaryConnectionLimit: runtimeHardCap - 100, + normalAdmissionPause: runtimeHardCap - 160 + }, + runtime: { + totalConnections: active, + preAuthConnections, + controls, + splices, + pendingSplices, + queuedBytes, + ...(legacy ? {} : { + inFlightConnections, + reservedConnectionUnits, + enforcedConnectionUnits + }) + } + }) + } + return response({ + status: { + cellId: config.cellId, + cellUrl: config.cellOrigin, + admissionState: admission === 'non-general' ? 'migration-only' : admission, + runtime: { heartbeatFresh, cellUrl: config.cellOrigin }, + assignments: 900, + activityLeases, + activityRequestUnits, + restartBlockingActivityLeases, + restartBlockingActivityRequestUnits, + restartBlockingReservedRequests, + reservedRequests, + outgoingMigrations, + incomingMigrations, + connectionCapacity: { + hardCap: directorHardCap, + unobservedBound: 60, + controlRebindReserve: 100, + ordinaryConnectionLimit: directorHardCap - 100, + normalAdmissionPause: directorHardCap - 160, + observedConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + pendingControlReservations, + heartbeatFresh + } + } + }) + } +} + +test('parses a paired reviewed capacity', () => { + assert.deepEqual( + parseCapacityTransitionArguments([ + '--director-origin', 'https://relay.example.com', + '--cell-origin', 'https://c3.relay.example.com', + '--cell-id', 'staging-gce-c3', + '--heartbeat', 'stale', + '--admission', 'migration-only', + '--draining', 'required', + '--activity', 'restart-safe', + '--expected-image-digests', imageDigest, + '--hard-cap', '1000', + '--unobserved-bound', '60' + ]), + { + ...config, + heartbeat: 'stale', + admission: 'migration-only', + activity: 'restart-safe', + runtime: 'required', + expectedImageDigests: [imageDigest], + timeoutMs: 180_000 + } + ) +}) + +test('accepts either exact predecessor image without weakening the live state checks', async () => { + const predecessor = `sha256:${'b'.repeat(64)}` + const compatible = `sha256:${'c'.repeat(64)}` + const predecessorConfig = parseCapacityTransitionArguments([ + '--director-origin', 'https://relay.example.com', + '--cell-origin', 'https://c3.relay.example.com', + '--cell-id', 'staging-gce-c3', + '--heartbeat', 'fresh', + '--admission', 'general', + '--draining', 'forbidden', + '--activity', 'allowed', + '--expected-image-digests', `${predecessor},${compatible}`, + '--hard-cap', '600', + '--unobserved-bound', '60' + ]) + assert.deepEqual(predecessorConfig.expectedImageDigests, [predecessor, compatible]) + assert.deepEqual( + { + heartbeat: predecessorConfig.heartbeat, + admission: predecessorConfig.admission, + draining: predecessorConfig.draining, + hardCap: predecessorConfig.hardCap, + unobservedBound: predecessorConfig.unobservedBound + }, + { + heartbeat: 'fresh', + admission: 'general', + draining: 'forbidden', + hardCap: 600, + unobservedBound: 60 + } + ) + const liveState = { + runtimeHardCap: 600, + directorHardCap: 600, + runtimeImageDigest: compatible, + admission: 'general', + draining: 'forbidden' + } + assert.equal( + (await verifyCapacityTransition(predecessorConfig, { + fetch: harness(liveState), + token: 'masked-token' + })).imageDigest, + compatible + ) + await assert.rejects( + verifyCapacityTransition(predecessorConfig, { + fetch: harness({ + ...liveState, + runtimeImageDigest: `sha256:${'d'.repeat(64)}` + }), + token: 'masked-token' + }), + /runtime does not match the cell/ + ) +}) + +test('requires the exact regional rehome protocol when requested', async () => { + const rehomeConfig = { + ...config, + regionalRehomeProtocol: 1 + } + await verifyCapacityTransition(rehomeConfig, { + token: 'token', + fetch: harness({ regionalRehomeProtocol: 1 }) + }) + await assert.rejects( + verifyCapacityTransition(rehomeConfig, { + token: 'token', + fetch: harness({ regionalRehomeProtocol: 0 }), + now: () => 1, + wait: async () => undefined + }), + /runtime does not match/ + ) + assert.equal( + parseCapacityTransitionArguments([ + '--director-origin', 'https://relay.example.com', + '--cell-origin', 'https://c3.relay.example.com', + '--cell-id', 'staging-gce-c3', + '--heartbeat', 'fresh', + '--admission', 'general', + '--draining', 'forbidden', + '--activity', 'allowed', + '--regional-rehome-protocol', '1' + ]).regionalRehomeProtocol, + 1 + ) +}) + +test('binds an absent rehome protocol to 0 for legacy pre-rehome images', async () => { + // A rolled-back cell runs an image that omits the field entirely; the + // documented contract binds absence to protocol 0. + await verifyCapacityTransition( + { ...config, regionalRehomeProtocol: 0 }, + { token: 'token', fetch: harness({ regionalRehomeProtocol: null }) } + ) + await verifyCapacityTransition( + { ...config, regionalRehomeProtocol: 0 }, + { token: 'token', fetch: harness({ regionalRehomeProtocol: 0 }) } + ) + await assert.rejects( + verifyCapacityTransition( + { ...config, regionalRehomeProtocol: 1 }, + { + token: 'token', + fetch: harness({ regionalRehomeProtocol: null }), + now: () => 1, + wait: async () => undefined + } + ), + /runtime does not match/ + ) +}) + +test('restart-safe mode requires the exact isolated drain state', () => { + assert.throws( + () => parseCapacityTransitionArguments([ + '--director-origin', 'https://relay.example.com', + '--cell-origin', 'https://c3.relay.example.com', + '--cell-id', 'staging-gce-c3', + '--heartbeat', 'either', + '--admission', 'general', + '--draining', 'required', + '--activity', 'restart-safe' + ]), + /requires migration-only admission and draining/ + ) +}) + +test('offline rollback requires two stale zero-durable-activity samples', async () => { + const offlineConfig = { + ...config, + heartbeat: 'stale', + admission: 'migration-only', + draining: 'either', + activity: 'restart-safe', + runtime: 'unavailable', + expectedImageDigests: [], + hardCap: undefined, + unobservedBound: undefined, + timeoutMs: 10_000 + } + const base = harness({ + heartbeatFresh: false, + admission: 'migration-only', + draining: 'forbidden', + activityLeases: 0, + activityRequestUnits: 0, + reservedRequests: 0, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: -1 + }) + let runtimeReads = 0 + let directorReads = 0 + let now = 0 + const result = await verifyCapacityTransition(offlineConfig, { + fetch: async (url, options) => { + const pathname = new URL(url).pathname + if (pathname === '/v1/admin/runtime-status') { + runtimeReads += 1 + return Response.json({ error: 'backend_unavailable' }, { status: 503 }) + } + if (pathname === '/v1/admin/cell-status') directorReads += 1 + return await base(url, options) + }, + token: 'masked-token', + now: () => now, + wait: async (milliseconds) => { + now += milliseconds + } + }) + assert.equal(result.cellId, config.cellId) + assert.equal(result.hardCap, null) + assert.equal(result.restartBlockingReservedRequests, -1) + assert.equal(runtimeReads, 2) + assert.equal(directorReads, 2) +}) + +test('offline rollback rejects a reachable cell or durable work', async () => { + const offlineConfig = { + ...config, + heartbeat: 'stale', + admission: 'migration-only', + draining: 'either', + activity: 'restart-safe', + runtime: 'unavailable', + expectedImageDigests: [], + hardCap: undefined, + unobservedBound: undefined, + timeoutMs: 0 + } + await assert.rejects( + verifyCapacityTransition(offlineConfig, { + fetch: harness({ heartbeatFresh: false, admission: 'migration-only' }), + token: 'masked-token' + }), + /timed out/ + ) + const active = harness({ + heartbeatFresh: false, + admission: 'migration-only', + activityLeases: 1 + }) + await assert.rejects( + verifyCapacityTransition(offlineConfig, { + fetch: async (url, options) => + new URL(url).pathname === '/v1/admin/runtime-status' + ? Response.json({ error: 'backend_unavailable' }, { status: 503 }) + : await active(url, options), + token: 'masked-token' + }), + /timed out/ + ) + await assert.rejects( + verifyCapacityTransition(offlineConfig, { + fetch: async (url, options) => { + if (new URL(url).pathname === '/v1/admin/runtime-status') { + return Response.json({ error: 'backend_unavailable' }, { status: 503 }) + } + const result = await active(url, options) + if (new URL(url).pathname !== '/v1/admin/cell-status') return result + const body = await result.json() + body.status.cellId = 'production-gce-other' + return response(body) + }, + token: 'masked-token' + }), + /does not match the cell/ + ) +}) + +test('offline rollback arguments cannot claim live capacity', () => { + assert.throws( + () => parseCapacityTransitionArguments([ + '--director-origin', 'https://relay.example.com', + '--cell-origin', 'https://c3.relay.example.com', + '--cell-id', 'staging-gce-c3', + '--heartbeat', 'stale', + '--admission', 'migration-only', + '--draining', 'either', + '--activity', 'restart-safe', + '--runtime', 'unavailable', + '--hard-cap', '600', + '--unobserved-bound', '60' + ]), + /cannot prove live capacity/ + ) +}) + +test('accepts a quiescent matching cell without exposing assignments', async () => { + assert.deepEqual( + await verifyCapacityTransition(config, { + fetch: harness(), + token: 'masked-token' + }), + { + cellId: config.cellId, + admissionState: 'migration-only', + assignments: 900, + hardCap: 1_000, + unobservedBound: 60, + heartbeatFresh: true, + imageDigest + } + ) +}) + +test('migration-only admission rejects the irreversible existing-only state', async () => { + const migrationConfig = { ...config, admission: 'migration-only' } + await verifyCapacityTransition(migrationConfig, { + fetch: harness({ admission: 'migration-only' }), + token: 'masked-token' + }) + await assert.rejects( + verifyCapacityTransition(migrationConfig, { + fetch: harness({ admission: 'existing-only' }), + token: 'masked-token' + }), + /admission does not match/ + ) +}) + +test('requires the exact runtime image and both cell origins', async () => { + await assert.rejects( + verifyCapacityTransition(config, { + fetch: harness({ runtimeImageDigest: `sha256:${'b'.repeat(64)}` }), + token: 'masked-token' + }), + /runtime does not match the cell/ + ) + const base = harness() + for (const location of ['runtime', 'director']) { + await assert.rejects( + verifyCapacityTransition(config, { + fetch: async (url, options) => { + const result = await base(url, options) + const path = new URL(url).pathname + if ( + (location === 'runtime' && path !== '/v1/admin/runtime-status') || + (location === 'director' && path !== '/v1/admin/cell-status') + ) return result + const body = await result.json() + if (location === 'runtime') body.cellUrl = 'https://other.example.com' + else body.status.cellUrl = 'https://other.example.com' + return response(body) + }, + token: 'masked-token' + }), + /does not match the cell/ + ) + } +}) + +test('accepts a quiescent legacy runtime before its first cap transition', async () => { + assert.equal( + ( + await verifyCapacityTransition( + { ...config, hardCap: undefined, unobservedBound: undefined, heartbeat: 'either' }, + { fetch: harness({ legacy: true }), token: 'masked-token' } + ) + ).cellId, + config.cellId + ) +}) + +test('uses the legacy runtime heartbeat when capacity telemetry is not registered', async () => { + const result = await verifyCapacityTransition( + { + ...config, + hardCap: undefined, + unobservedBound: undefined, + heartbeat: 'fresh', + draining: 'forbidden', + activity: 'allowed' + }, + { fetch: harness({ legacy: true, draining: 'forbidden' }), token: 'masked-token' } + ) + assert.equal(result.heartbeatFresh, true) +}) + +test('rejects old directors and active cells', async () => { + await assert.rejects( + verifyCapacityTransition(config, { fetch: harness({ protocol: 1 }), token: 'masked' }), + /not capacity-protocol compatible/ + ) + let now = 0 + await assert.rejects( + verifyCapacityTransition(config, { + fetch: harness({ active: 1 }), + token: 'masked', + now: () => now, + wait: async (milliseconds) => { + now += milliseconds + } + }), + /timed out/ + ) +}) + +test('accepts active controls only when the transition explicitly allows them', async () => { + const activeConfig = { + ...config, + admission: 'general', + draining: 'forbidden', + activity: 'allowed' + } + const result = await verifyCapacityTransition(activeConfig, { + fetch: harness({ active: 900, admission: 'general', draining: 'forbidden' }), + token: 'masked' + }) + assert.equal(result.admissionState, 'general') +}) + +test('accepts only rejected reconnect traffic at the restart gate', async () => { + const restartConfig = { + ...config, + admission: 'migration-only', + activity: 'restart-safe', + timeoutMs: 10_000 + } + const reconnecting = harness({ + active: 10, + activityLeases: 839, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: 0, + pendingControlReservations: 839 + }) + let now = 0 + let runtimeReads = 0 + const fetch = async (url, options) => { + if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 + return await reconnecting(url, options) + } + assert.equal((await verifyCapacityTransition(restartConfig, { + fetch, + token: 'masked-token', + now: () => now, + wait: async (milliseconds) => { + now += milliseconds + } + })).cellId, config.cellId) + assert.equal(runtimeReads, 2) + await assert.rejects( + verifyCapacityTransition({ ...config, timeoutMs: 0 }, { + fetch: reconnecting, + token: 'masked-token' + }), + /timed out/ + ) +}) + +test('restart-safe settling resets after data-plane admission appears', async () => { + const restartConfig = { + ...config, + admission: 'migration-only', + activity: 'restart-safe', + timeoutMs: 20_000 + } + const safe = harness({ active: 1, activityLeases: 0 }) + const unsafe = harness({ active: 1, activityLeases: 0, preAuthConnections: 1 }) + let runtimeReads = 0 + let now = 0 + const result = await verifyCapacityTransition(restartConfig, { + fetch: async (url, options) => { + if (new URL(url).pathname !== '/v1/admin/runtime-status') { + return await safe(url, options) + } + runtimeReads += 1 + return await (runtimeReads === 2 ? unsafe : safe)(url, options) + }, + token: 'masked-token', + now: () => now, + wait: async (milliseconds) => { + now += milliseconds + } + }) + assert.equal(result.cellId, config.cellId) + assert.equal(runtimeReads, 4) +}) + +test('restart-safe settling resets after director activity appears', async () => { + const restartConfig = { + ...config, + admission: 'migration-only', + activity: 'restart-safe', + timeoutMs: 20_000 + } + const safe = harness({ + activityLeases: 839, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: 0 + }) + const unsafe = harness({ + activityLeases: 840, + restartBlockingActivityLeases: 1, + restartBlockingActivityRequestUnits: 1, + restartBlockingReservedRequests: 1 + }) + let directorReads = 0 + let runtimeReads = 0 + let now = 0 + const result = await verifyCapacityTransition(restartConfig, { + fetch: async (url, options) => { + if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 + if (new URL(url).pathname !== '/v1/admin/cell-status') { + return await safe(url, options) + } + directorReads += 1 + return await (directorReads === 2 ? unsafe : safe)(url, options) + }, + token: 'masked-token', + now: () => now, + wait: async (milliseconds) => { + now += milliseconds + } + }) + assert.equal(result.cellId, config.cellId) + assert.equal(runtimeReads, 4) +}) + +test('restart gate rejects malformed restart aggregates', async (t) => { + const restartConfig = { + ...config, + admission: 'migration-only', + activity: 'restart-safe', + timeoutMs: 0 + } + for (const field of [ + 'restartBlockingActivityLeases', + 'restartBlockingActivityRequestUnits' + ]) { + for (const value of [undefined, -1]) { + await t.test(`${field} ${value === undefined ? 'missing' : 'negative'}`, async () => { + const base = harness({ [field]: value }) + const fetch = value === undefined + ? async (url, options) => { + const result = await base(url, options) + if (new URL(url).pathname !== '/v1/admin/cell-status') return result + const body = await result.json() + delete body.status[field] + return response(body) + } + : base + await assert.rejects( + verifyCapacityTransition(restartConfig, { + fetch, + token: 'masked-token' + }), + /is invalid/ + ) + }) + } + } + for (const value of [undefined, null, '0', false]) { + await t.test(`restartBlockingReservedRequests ${String(value)}`, async () => { + const base = harness({ + preAuthConnections: 1, + restartBlockingActivityLeases: 1, + restartBlockingReservedRequests: value + }) + await assert.rejects( + verifyCapacityTransition(restartConfig, { + fetch: async (url, options) => { + const result = await base(url, options) + if ( + value !== undefined || + new URL(url).pathname !== '/v1/admin/cell-status' + ) return result + const body = await result.json() + delete body.status.restartBlockingReservedRequests + return response(body) + }, + token: 'masked-token' + }), + /is invalid/ + ) + }) + } +}) + +test('offline rollback validates restart reservation accounting before other blockers', async (t) => { + const offlineConfig = { + ...config, + heartbeat: 'stale', + admission: 'migration-only', + draining: 'either', + activity: 'restart-safe', + runtime: 'unavailable', + expectedImageDigests: [], + hardCap: undefined, + unobservedBound: undefined, + timeoutMs: 0 + } + for (const value of [undefined, null, '0', false]) { + await t.test(String(value), async () => { + const base = harness({ + heartbeatFresh: false, + activityLeases: 1, + restartBlockingReservedRequests: value + }) + await assert.rejects( + verifyCapacityTransition(offlineConfig, { + fetch: async (url, options) => { + const pathname = new URL(url).pathname + if (pathname === '/v1/admin/runtime-status') { + return Response.json({ error: 'backend_unavailable' }, { status: 503 }) + } + const result = await base(url, options) + if (value !== undefined || pathname !== '/v1/admin/cell-status') return result + const body = await result.json() + delete body.status.restartBlockingReservedRequests + return response(body) + }, + token: 'masked-token' + }), + /is invalid/ + ) + }) + } +}) + +test('negative restart reservation accounting is safe and remains observable', async () => { + const result = await verifyCapacityTransition( + { + ...config, + admission: 'migration-only', + activity: 'restart-safe', + timeoutMs: 10_000 + }, + { + fetch: harness({ restartBlockingReservedRequests: -1 }), + token: 'masked-token', + now: () => 0, + wait: async () => undefined + } + ) + assert.equal(result.restartBlockingReservedRequests, -1) +}) + +test('negative restart reservation accounting cannot mask real work', async () => { + await assert.rejects( + verifyCapacityTransition( + { + ...config, + admission: 'migration-only', + activity: 'restart-safe', + timeoutMs: 0 + }, + { + fetch: harness({ + restartBlockingActivityLeases: 1, + restartBlockingReservedRequests: -1 + }), + token: 'masked-token' + } + ), + /"restartBlockingReservedRequests":-1/ + ) +}) + +test('restart gate rejects live or durable cell work', async (t) => { + const restartConfig = { + ...config, + admission: 'migration-only', + activity: 'restart-safe', + timeoutMs: 0 + } + const unsafeStates = [ + ['control', { active: 1, activityLeases: 0, controls: 1 }], + ['splice', { active: 1, activityLeases: 0, splices: 1 }], + ['pending splice', { active: 1, activityLeases: 0, pendingSplices: 1 }], + ['queued data', { active: 1, activityLeases: 0, queuedBytes: 1 }], + ['pre-auth connection', { active: 1, activityLeases: 0, preAuthConnections: 1 }], + ['in-flight connection', { + activityLeases: 0, + inFlightConnections: 1, + enforcedConnectionUnits: 1 + }], + ['reserved data unit', { + active: 1, + activityLeases: 0, + reservedConnectionUnits: 1, + enforcedConnectionUnits: 2 + }], + ['activity lease', { activityLeases: 1 }], + ['misaccounted pending control lease', { + activityLeases: 1, + activityRequestUnits: 2, + reservedRequests: 2, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 1, + restartBlockingReservedRequests: 1 + }], + ['cell reservation', { reservedRequests: 1 }], + ['outgoing migration', { outgoingMigrations: 1 }], + ['incoming migration', { incomingMigrations: 1 }] + ] + for (const [name, state] of unsafeStates) { + await t.test(name, async () => { + await assert.rejects( + verifyCapacityTransition(restartConfig, { + fetch: harness(state), + token: 'masked-token' + }), + /timed out/ + ) + }) + } +}) + +test('restart timeout reports only aggregate blockers', async () => { + const base = harness({ + controls: 2, + restartBlockingActivityLeases: 3, + restartBlockingActivityRequestUnits: 4, + restartBlockingReservedRequests: 5, + incomingMigrations: 6 + }) + await assert.rejects( + verifyCapacityTransition( + { + ...config, + admission: 'migration-only', + draining: 'required', + activity: 'restart-safe', + timeoutMs: 0 + }, + { fetch: base, token: 'masked-token' } + ), + (error) => { + assert.match(error.message, /"controls":2/) + assert.match(error.message, /"restartBlockingActivityLeases":3/) + assert.match(error.message, /"incomingMigrations":6/) + assert.doesNotMatch(error.message, /user|host|secret|token/i) + return true + } + ) +}) + +test('restart timeout reports one of two safe settling samples', async () => { + const base = harness() + await assert.rejects( + verifyCapacityTransition( + { + ...config, + admission: 'migration-only', + draining: 'required', + activity: 'restart-safe', + timeoutMs: 0 + }, + { fetch: base, token: 'masked-token' } + ), + (error) => { + assert.match(error.message, /"restartSafeSamples":1/) + assert.match(error.message, /"requiredRestartSafeSamples":2/) + return true + } + ) +}) + +test('quiescent timeout reports connection and capacity aggregates', async () => { + const base = harness({ + active: 2, + enforcedConnectionUnits: 3, + activityLeases: 0, + activityRequestUnits: 0, + reservedRequests: 0, + restartBlockingActivityLeases: 0, + restartBlockingActivityRequestUnits: 0, + restartBlockingReservedRequests: 0, + pendingControlReservations: 4, + heartbeatFresh: false + }) + await assert.rejects( + verifyCapacityTransition({ ...config, timeoutMs: 0 }, { fetch: base, token: 'masked-token' }), + (error) => { + assert.match(error.message, /"totalConnections":2/) + assert.match(error.message, /"enforcedConnectionUnits":3/) + assert.match(error.message, /"pendingControlReservations":4/) + assert.match(error.message, /"heartbeatFresh":false/) + return true + } + ) +}) + +test('timeout distinguishes runtime and director capacity views', async () => { + const base = harness({ runtimeHardCap: 999 }) + await assert.rejects( + verifyCapacityTransition({ ...config, timeoutMs: 0 }, { fetch: base, token: 'masked-token' }), + (error) => { + assert.match(error.message, /"runtimeCapacity":\{"hardCap":999/) + assert.match(error.message, /"directorCapacity":\{"hardCap":1000/) + return true + } + ) +}) + +test('offline timeout reports every durable activity aggregate', async () => { + const base = harness({ + activityLeases: 1, + activityRequestUnits: 2, + reservedRequests: 3, + pendingControlReservations: 4, + heartbeatFresh: false + }) + await assert.rejects( + verifyCapacityTransition( + { + ...config, + admission: 'migration-only', + draining: 'either', + activity: 'restart-safe', + heartbeat: 'stale', + runtime: 'unavailable', + hardCap: undefined, + unobservedBound: undefined, + timeoutMs: 0 + }, + { + fetch: async (url, options) => + new URL(url).pathname === '/v1/admin/runtime-status' + ? Response.json({ error: 'backend_unavailable' }, { status: 503 }) + : await base(url, options), + token: 'masked-token' + } + ), + (error) => { + assert.match(error.message, /"activityLeases":1/) + assert.match(error.message, /"activityRequestUnits":2/) + assert.match(error.message, /"reservedRequests":3/) + assert.match(error.message, /"pendingControlReservations":4/) + return true + } + ) +}) + +test('waits for a drained runtime to become quiescent', async () => { + let runtimeReads = 0 + const base = harness() + const fetch = async (url, options) => { + if (new URL(url).pathname === '/v1/admin/runtime-status' && runtimeReads++ === 0) { + return Response.json({ + role: 'cell', + cellId: config.cellId, + cellUrl: config.cellOrigin, + imageDigest, + draining: true, + connectionCapacity: { + hardCap: 1_000, + unobservedBound: 60, + controlRebindReserve: 100, + ordinaryConnectionLimit: 900, + normalAdmissionPause: 840 + }, + runtime: { totalConnections: 1, preAuthConnections: 0, enforcedConnectionUnits: 1 } + }) + } + return await base(url, options) + } + let now = 0 + const result = await verifyCapacityTransition( + { ...config, timeoutMs: 10_000 }, + { + fetch, + token: 'masked', + now: () => now, + wait: async (milliseconds) => { + now += milliseconds + } + } + ) + assert.equal(result.cellId, config.cellId) + assert.equal(runtimeReads, 2) +}) + +test('waits for transient cell routing after a replacement becomes stable', async () => { + for (const status of [502, 503, 504]) { + let runtimeReads = 0 + let unavailableResponse + const base = harness() + const fetch = async (url, options) => { + if (new URL(url).pathname === '/v1/admin/runtime-status' && runtimeReads++ === 0) { + unavailableResponse = Response.json({ error: 'backend_unavailable' }, { status }) + return unavailableResponse + } + return await base(url, options) + } + let now = 0 + const result = await verifyCapacityTransition( + { ...config, timeoutMs: 10_000 }, + { + fetch, + token: 'masked', + now: () => now, + wait: async (milliseconds) => { + now += milliseconds + } + } + ) + assert.equal(result.cellId, config.cellId) + assert.equal(runtimeReads, 2) + assert.equal(unavailableResponse.bodyUsed, true) + } +}) + +test('persistent cell unavailability fails closed at the deadline', async () => { + let runtimeReads = 0 + let directorStatusReads = 0 + let waits = 0 + let now = 0 + const base = harness() + await assert.rejects( + verifyCapacityTransition( + { ...config, timeoutMs: 10_000 }, + { + fetch: async (url, options) => { + const pathname = new URL(url).pathname + if (pathname === '/v1/admin/runtime-status') { + runtimeReads += 1 + return Response.json({ error: 'backend_unavailable' }, { status: 503 }) + } + if (pathname === '/v1/admin/cell-status') directorStatusReads += 1 + return await base(url, options) + }, + token: 'masked', + now: () => now, + wait: async (milliseconds) => { + waits += 1 + now += milliseconds + } + } + ), + /capacity transition verification timed out: \{"runtimeAvailable":false\}/ + ) + assert.equal(runtimeReads, 3) + assert.equal(directorStatusReads, 0) + assert.equal(waits, 2) +}) + +test('general-or-migration-only admits both recovery states and nothing else', async () => { + for (const [admission, accepted] of [ + ['general', true], + ['migration-only', true], + ['existing-only', false] + ]) { + const attempt = verifyCapacityTransition( + { + ...config, + admission: 'general-or-migration-only', + draining: 'either', + activity: 'allowed' + }, + { fetch: harness({ admission, draining: 'either' }), token: 'masked' } + ) + if (accepted) { + await attempt + } else { + await assert.rejects(attempt, /admission does not match/) + } + } +}) + +test('does not retry a rejected cell admin token', async () => { + let waits = 0 + const base = harness() + await assert.rejects( + verifyCapacityTransition(config, { + fetch: async (url, options) => + new URL(url).pathname === '/v1/admin/runtime-status' + ? Response.json({ error: 'invalid_token' }, { status: 401 }) + : await base(url, options), + token: 'masked', + wait: async () => { + waits += 1 + } + }), + /cell runtime status returned 401/ + ) + assert.equal(waits, 0) +}) diff --git a/cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs b/cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs new file mode 100644 index 00000000000..1c51ac28f36 --- /dev/null +++ b/cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs @@ -0,0 +1,292 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +const CAPACITY_PROTOCOL = 2 +const LEGACY_RUNTIME_KEYS = ['cellId', 'cellUrl', 'imageDigest', 'role', 'v'] +const METRIC_COUNTS = [ + 'totalConnections', + 'preAuthConnections', + 'controls', + 'splices', + 'pendingSplices', + 'queuedBytes' +] + +function integer(value, name) { + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} is invalid`) + return value +} + +export function parseLegacyBootstrapArguments(argv) { + const values = {} + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index] + const value = argv[index + 1] + if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') + values[key.slice(2)] = value + } + for (const key of [ + 'director-origin', + 'cell-origin', + 'cell-id', + 'admission', + 'expected-image-digest', + 'metrics-after', + 'metrics-file' + ]) { + if (!values[key]) throw new Error(`missing --${key}`) + } + if (!['general', 'migration-only'].includes(values.admission)) { + throw new Error('--admission must be general or migration-only') + } + const directorOrigin = new URL(values['director-origin']) + const cellOrigin = new URL(values['cell-origin']) + if ( + directorOrigin.protocol !== 'https:' || + directorOrigin.origin !== values['director-origin'] || + cellOrigin.protocol !== 'https:' || + cellOrigin.origin !== values['cell-origin'] + ) { + throw new Error('origins must be canonical HTTPS origins') + } + if (!/^sha256:[a-f0-9]{64}$/.test(values['expected-image-digest'])) { + throw new Error('--expected-image-digest is invalid') + } + const metricsAfter = Date.parse(values['metrics-after']) + if (!Number.isFinite(metricsAfter)) throw new Error('--metrics-after is invalid') + const runtimeStartedAfter = values['runtime-started-after'] === undefined + ? undefined + : Date.parse(values['runtime-started-after']) + if (runtimeStartedAfter !== undefined && !Number.isFinite(runtimeStartedAfter)) { + throw new Error('--runtime-started-after is invalid') + } + const previousIncarnationDigest = values['previous-incarnation-digest'] + if ( + previousIncarnationDigest !== undefined && + !/^[a-f0-9]{64}$/.test(previousIncarnationDigest) + ) { + throw new Error('--previous-incarnation-digest is invalid') + } + const hardCap = values['hard-cap'] === undefined + ? undefined + : integer(Number(values['hard-cap']), '--hard-cap') + const unobservedBound = values['unobserved-bound'] === undefined + ? undefined + : integer(Number(values['unobserved-bound']), '--unobserved-bound') + if ((hardCap === undefined) !== (unobservedBound === undefined)) { + throw new Error('capacity expectations must be paired') + } + const capacityState = values['capacity-state'] ?? (hardCap === undefined ? 'absent' : 'stale') + if (!['absent', 'stale', 'absent-or-stale'].includes(capacityState)) { + throw new Error('--capacity-state is invalid') + } + if ((capacityState === 'absent') !== (hardCap === undefined)) { + throw new Error('capacity state and expectations do not match') + } + return { + directorOrigin: directorOrigin.origin, + cellOrigin: cellOrigin.origin, + cellId: values['cell-id'], + admission: values.admission, + expectedImageDigest: values['expected-image-digest'], + metricsAfter, + metricsFile: values['metrics-file'], + runtimeStartedAfter, + previousIncarnationDigest, + hardCap, + unobservedBound, + capacityState + } +} + +async function responseJson(response, label) { + const body = await response.json().catch(() => ({})) + if (!response.ok) throw new Error(`${label} returned ${response.status}`) + return body +} + +function requireLegacyRuntime(runtime, config) { + if (JSON.stringify(Object.keys(runtime).sort()) !== JSON.stringify(LEGACY_RUNTIME_KEYS)) { + throw new Error('cell runtime does not match the reviewed legacy contract') + } + if ( + runtime.v !== 1 || + runtime.role !== 'cell' || + runtime.cellId !== config.cellId || + runtime.cellUrl !== config.cellOrigin || + runtime.imageDigest !== config.expectedImageDigest + ) { + throw new Error('legacy cell runtime identity does not match') + } +} + +function requireDirectorQuiescence(status, config, now) { + const capacity = status.connectionCapacity + const staleCapacityTransition = config.capacityState !== 'absent' && capacity !== null + if ( + status.cellId !== config.cellId || + status.cellUrl !== config.cellOrigin || + status.enabled !== true || + status.admissionState !== config.admission || + status.runtime?.ready !== true || + (status.runtime.heartbeatFresh !== true && + !(staleCapacityTransition && status.runtime.heartbeatFresh === false)) || + status.runtime.cellUrl !== config.cellOrigin + ) { + throw new Error('legacy cell is not fresh, ready, and in the required admission state') + } + if (config.capacityState === 'absent' && capacity !== null) { + throw new Error('legacy cell has unexpected connection capacity') + } + if ( + config.capacityState !== 'absent' && + !(config.capacityState === 'absent-or-stale' && capacity === null) && + (capacity?.hardCap !== config.hardCap || + capacity.unobservedBound !== config.unobservedBound || + capacity.controlRebindReserve !== 100 || + capacity.ordinaryConnectionLimit !== config.hardCap - 100 || + capacity.normalAdmissionPause !== config.hardCap - 100 - config.unobservedBound || + capacity.heartbeatFresh !== false) + ) { + throw new Error('legacy cell connection capacity does not match') + } + integer(status.runtime.startedAt, 'runtime started at') + integer(status.runtime.lastHeartbeatAt, 'runtime heartbeat at') + if (!/^[0-9a-f-]{36}$/.test(status.runtime.cellIncarnation)) { + throw new Error('runtime cell incarnation is invalid') + } + const incarnationDigest = createHash('sha256') + .update(status.runtime.cellIncarnation) + .digest('hex') + if (incarnationDigest === config.previousIncarnationDigest) { + throw new Error('legacy fallback heartbeat incarnation did not change') + } + if ( + config.runtimeStartedAfter !== undefined && + (integer(status.runtime.startedAt, 'runtime started at') < config.runtimeStartedAfter || + status.runtime.startedAt > now + 30_000) + ) { + throw new Error('legacy fallback heartbeat predates the replacement') + } + const activity = [ + status.reservedRequests, + status.activityLeases, + status.activityRequestUnits, + status.outgoingMigrations, + status.incomingMigrations, + status.runtime.observedRequests + ] + if (capacity !== null) { + activity.push( + capacity.observedConnections, + capacity.inFlightConnections, + capacity.reservedConnectionUnits, + capacity.enforcedConnectionUnits, + capacity.pendingControlReservations + ) + } + if (activity.some((value) => integer(value, 'director activity count') !== 0)) { + throw new Error('legacy fallback has durable activity') + } + integer(status.assignments, 'assignments') + return incarnationDigest +} + +function requireFreshZeroMetrics(metrics, config, now) { + if (!Array.isArray(metrics)) throw new Error('legacy runtime metrics are invalid') + const samples = metrics.filter((entry) => { + const timestamp = Date.parse(entry?.timestamp) + return Number.isFinite(timestamp) && timestamp >= config.metricsAfter && timestamp <= now + 30_000 + }) + const timestamps = new Set(samples.map((entry) => entry.timestamp)) + if (samples.length < 2 || timestamps.size < 2) { + throw new Error('legacy runtime metrics need two post-boundary samples') + } + const latest = Math.max(...samples.map((entry) => Date.parse(entry.timestamp))) + if (latest < now - 90_000) throw new Error('legacy runtime metrics are stale') + for (const sample of samples) { + if (sample.cellId !== config.cellId || sample.metricVersion !== 1) { + throw new Error('legacy runtime metrics do not match the cell') + } + if (METRIC_COUNTS.some((field) => integer(sample[field], field) !== 0)) { + throw new Error('legacy runtime metrics are not quiescent') + } + } + return samples.length +} + +export async function verifyLegacyBootstrap(config, overrides = {}) { + const fetchImpl = overrides.fetch ?? fetch + const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN + const now = overrides.now?.() ?? Date.now() + const metrics = overrides.metrics ?? JSON.parse(readFileSync(config.metricsFile, 'utf8')) + if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') + const publicChecks = await Promise.all([ + responseJson( + await fetchImpl(`${config.directorOrigin}/health`, { + signal: AbortSignal.timeout(15_000) + }), + 'director health' + ), + responseJson( + await fetchImpl(`${config.cellOrigin}/health`, { signal: AbortSignal.timeout(15_000) }), + 'cell health' + ), + responseJson( + await fetchImpl(`${config.cellOrigin}/ready`, { signal: AbortSignal.timeout(15_000) }), + 'cell readiness' + ) + ]) + if ( + publicChecks[0].ok !== true || + publicChecks[0].connectionCapacityProtocol !== CAPACITY_PROTOCOL || + publicChecks[1].ok !== true || + publicChecks[2].ok !== true + ) { + throw new Error('legacy fallback public checks failed') + } + const headers = { authorization: `Bearer ${token}`, 'content-type': 'application/json' } + const [runtime, result] = await Promise.all([ + responseJson( + await fetchImpl(`${config.cellOrigin}/v1/admin/runtime-status`, { + method: 'POST', + headers, + body: JSON.stringify({ v: 1 }), + signal: AbortSignal.timeout(30_000) + }), + 'cell runtime status' + ), + responseJson( + await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, { + method: 'POST', + headers, + body: JSON.stringify({ v: 1, cellId: config.cellId }), + signal: AbortSignal.timeout(30_000) + }), + 'cell status' + ) + ]) + requireLegacyRuntime(runtime, config) + const incarnationDigest = requireDirectorQuiescence(result.status, config, now) + const metricSamples = requireFreshZeroMetrics(metrics, config, now) + return { + cellId: config.cellId, + admissionState: result.status.admissionState, + assignments: integer(result.status.assignments, 'assignments'), + metricSamples, + incarnationDigest + } +} + +export async function main(argv = process.argv.slice(2)) { + const result = await verifyLegacyBootstrap(parseLegacyBootstrapArguments(argv)) + process.stdout.write(`${JSON.stringify({ event: 'relay_legacy_bootstrap_verified', ...result })}\n`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs b/cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs new file mode 100644 index 00000000000..0cb3cb5a25a --- /dev/null +++ b/cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs @@ -0,0 +1,395 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { relayWorkflowUrl } from './relay-repository.mjs' +import { + parseLegacyBootstrapArguments, + verifyLegacyBootstrap +} from './verify-relay-legacy-bootstrap.mjs' + +const now = Date.parse('2026-08-10T22:10:00Z') +const digest = `sha256:${'a'.repeat(64)}` +const config = { + directorOrigin: 'https://relay.example.com', + cellOrigin: 'https://c3.relay.example.com', + cellId: 'staging-gce-c3', + admission: 'general', + expectedImageDigest: digest, + metricsAfter: now - 120_000, + metricsFile: '/unused', + runtimeStartedAfter: undefined, + previousIncarnationDigest: undefined, + hardCap: undefined, + unobservedBound: undefined, + capacityState: 'absent' +} + +const metrics = [30, 60].map((secondsAgo) => ({ + timestamp: new Date(now - secondsAgo * 1_000).toISOString(), + cellId: config.cellId, + metricVersion: 1, + totalConnections: 0, + preAuthConnections: 0, + controls: 0, + splices: 0, + pendingSplices: 0, + queuedBytes: 0 +})) + +function response(body, status = 200) { + return Response.json(body, { status }) +} + +function harness({ + runtime = {}, + status = {}, + ready = true, + cell = config, + runtimeHeartbeatFresh = true +} = {}) { + return async (url) => { + const path = new URL(url).pathname + if (path === '/health') { + return response( + url.startsWith(config.directorOrigin) + ? { ok: true, connectionCapacityProtocol: 2 } + : { ok: true } + ) + } + if (path === '/ready') return ready ? response({ ok: true }) : response({ error: 'no' }, 503) + if (path === '/v1/admin/runtime-status') { + return response({ + v: 1, + role: 'cell', + cellId: cell.cellId, + cellUrl: cell.cellOrigin, + imageDigest: digest, + ...runtime + }) + } + return response({ + status: { + cellId: cell.cellId, + cellUrl: cell.cellOrigin, + enabled: true, + admissionState: 'general', + assignments: 12, + reservedRequests: 0, + activityLeases: 0, + activityRequestUnits: 0, + outgoingMigrations: 0, + incomingMigrations: 0, + connectionCapacity: null, + runtime: { + cellUrl: cell.cellOrigin, + cellIncarnation: '00000000-0000-4000-8000-000000000001', + startedAt: now - 90_000, + ready: true, + observedRequests: 0, + lastHeartbeatAt: now - 1_000, + heartbeatFresh: runtimeHeartbeatFresh + }, + ...status + } + }) + } +} + +test('parses the exact legacy bootstrap evidence boundary', () => { + assert.deepEqual( + parseLegacyBootstrapArguments([ + '--director-origin', config.directorOrigin, + '--cell-origin', config.cellOrigin, + '--cell-id', config.cellId, + '--admission', 'general', + '--expected-image-digest', digest, + '--metrics-after', new Date(config.metricsAfter).toISOString(), + '--metrics-file', '/tmp/metrics.json' + ]), + { ...config, metricsFile: '/tmp/metrics.json' } + ) +}) + +test('accepts the exact old runtime shape with two fresh zero samples', async () => { + const result = await verifyLegacyBootstrap(config, { + fetch: harness(), + metrics, + now: () => now, + token: 'masked-token' + }) + assert.deepEqual( + { ...result, incarnationDigest: undefined }, + { + cellId: config.cellId, + admissionState: 'general', + assignments: 12, + metricSamples: 2, + incarnationDigest: undefined + } + ) + assert.match(result.incarnationDigest, /^[a-f0-9]{64}$/) +}) + +test('rejects a new or unknown runtime shape on the legacy-only path', async () => { + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness({ runtime: { draining: false } }), + metrics, + now: () => now, + token: 'masked-token' + }), + /reviewed legacy contract/ + ) +}) + +test('rejects active durable state and active runtime metrics', async () => { + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness({ status: { activityLeases: 1 } }), + metrics, + now: () => now, + token: 'masked-token' + }), + /durable activity/ + ) + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness(), + metrics: metrics.map((entry, index) => ({ + ...entry, + controls: index === 0 ? 1 : 0 + })), + now: () => now, + token: 'masked-token' + }), + /not quiescent/ + ) + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness(), + metrics: metrics.map((entry) => ({ ...entry, pendingSplices: null })), + now: () => now, + token: 'masked-token' + }), + /pendingSplices is invalid/ + ) +}) + +test('rejects stale, pre-boundary, or single runtime samples', async () => { + for (const invalidMetrics of [ + metrics.map((entry) => ({ ...entry, timestamp: new Date(now - 180_000).toISOString() })), + [metrics[0]], + metrics.map((entry) => ({ ...entry, timestamp: new Date(now - 100_000).toISOString() })) + ]) { + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness(), + metrics: invalidMetrics, + now: () => now, + token: 'masked-token' + }), + /samples|stale/ + ) + } +}) + +test('rejects the wrong legacy image and an unready cell', async () => { + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness({ runtime: { imageDigest: `sha256:${'b'.repeat(64)}` } }), + metrics, + now: () => now, + token: 'masked-token' + }), + /identity does not match/ + ) + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness({ ready: false }), + metrics, + now: () => now, + token: 'masked-token' + }), + /readiness returned 503/ + ) +}) + +test('binds the director heartbeat to the replacement boundary', async () => { + const replacementConfig = { ...config, runtimeStartedAfter: now - 60_000 } + await assert.rejects( + verifyLegacyBootstrap(replacementConfig, { + fetch: harness(), + metrics, + now: () => now, + token: 'masked-token' + }), + /heartbeat predates/ + ) + await verifyLegacyBootstrap(replacementConfig, { + fetch: harness({ status: { runtime: { + cellUrl: config.cellOrigin, + cellIncarnation: '00000000-0000-4000-8000-000000000002', + startedAt: now - 30_000, + ready: true, + observedRequests: 0, + lastHeartbeatAt: now - 1_000, + heartbeatFresh: true + } } }), + metrics, + now: () => now, + token: 'masked-token' + }) +}) + +test('accepts a drained migration-only legacy target', async () => { + const migrationConfig = { ...config, admission: 'migration-only' } + const result = await verifyLegacyBootstrap(migrationConfig, { + fetch: harness({ status: { admissionState: 'migration-only' } }), + metrics, + now: () => now, + token: 'masked-token' + }) + assert.equal(result.admissionState, 'migration-only') +}) + +test('accepts exact stale capacity during the director-first transition', async () => { + const capacity = { + hardCap: 600, + unobservedBound: 60, + controlRebindReserve: 100, + ordinaryConnectionLimit: 500, + normalAdmissionPause: 440, + observedConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + pendingControlReservations: 0, + heartbeatFresh: false + } + const transition = { + ...config, + admission: 'migration-only', + hardCap: 600, + unobservedBound: 60, + capacityState: 'stale' + } + await verifyLegacyBootstrap(transition, { + fetch: harness({ status: { admissionState: 'migration-only', connectionCapacity: capacity } }), + metrics, + now: () => now, + token: 'masked-token' + }) + await assert.rejects( + verifyLegacyBootstrap(transition, { + fetch: harness({ status: { + admissionState: 'migration-only', + connectionCapacity: { ...capacity, heartbeatFresh: true } + } }), + metrics, + now: () => now, + token: 'masked-token' + }), + /capacity does not match/ + ) +}) + +test('resumes either legacy cell after the director update', async () => { + const capacity = { + hardCap: 600, + unobservedBound: 60, + controlRebindReserve: 100, + ordinaryConnectionLimit: 500, + normalAdmissionPause: 440, + observedConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0, + pendingControlReservations: 0, + heartbeatFresh: false + } + for (const number of [2, 3]) { + const cell = { + ...config, + cellId: `staging-gce-c${number}`, + cellOrigin: `https://c${number}.relay.example.com`, + admission: 'migration-only', + hardCap: 600, + unobservedBound: 60, + capacityState: 'absent-or-stale' + } + const cellMetrics = metrics.map((entry) => ({ ...entry, cellId: cell.cellId })) + for (const connectionCapacity of [null, capacity]) { + await verifyLegacyBootstrap(cell, { + fetch: harness({ + cell, + runtimeHeartbeatFresh: connectionCapacity === null, + status: { admissionState: 'migration-only', connectionCapacity } + }), + metrics: cellMetrics, + now: () => now, + token: 'masked-token' + }) + } + } +}) + +test('requires a fresh director heartbeat before capacity is configured', async () => { + await assert.rejects( + verifyLegacyBootstrap(config, { + fetch: harness({ runtimeHeartbeatFresh: false }), + metrics, + now: () => now, + token: 'masked-token' + }), + /fresh, ready/ + ) +}) + +test('requires a new director heartbeat incarnation after restart', async () => { + const before = await verifyLegacyBootstrap(config, { + fetch: harness(), + metrics, + now: () => now, + token: 'masked-token' + }) + await assert.rejects( + verifyLegacyBootstrap( + { ...config, previousIncarnationDigest: before.incarnationDigest }, + { fetch: harness(), metrics, now: () => now, token: 'masked-token' } + ), + /incarnation did not change/ + ) +}) + +test('rejects same-instance samples written before restart completion', async () => { + await assert.rejects( + verifyLegacyBootstrap( + { ...config, metricsAfter: now - 20_000 }, + { fetch: harness(), metrics, now: () => now, token: 'masked-token' } + ), + /post-boundary samples/ + ) +}) + +test('the bootstrap workflow binds zero metrics to a replacement C3 instance', async () => { + const { readFile } = await import('node:fs/promises') + const workflow = await readFile( + relayWorkflowUrl('bootstrap-relay-staging-capacity.yml'), + 'utf8' + ) + assert.match(workflow, /resource\.labels\.instance_id=.*\$\{instance_id\}/) + assert.match(workflow, /timestamp>=.*\$\{after\}/) + assert.doesNotMatch(workflow, /legacy_c3_instance_id.*!=/) + const c2Proof = workflow.indexOf('staging-gce-c2 general "${legacy_pre_boundary}"') + const isolate = workflow.indexOf('--mode isolate', c2Proof) + const drainedProof = workflow.indexOf('staging-gce-c3 migration-only', isolate) + const recreate = workflow.indexOf('recreate-instances', drainedProof) + const replacementProof = workflow.indexOf('"${legacy_c3_old_incarnation}"', recreate) + const stable = workflow.indexOf('wait-until', recreate) + const metricsBoundary = workflow.indexOf('legacy_c3_metrics_boundary=', stable) + assert.ok(c2Proof < isolate && isolate < drainedProof && drainedProof < recreate) + assert.ok(recreate < stable && stable < metricsBoundary && metricsBoundary < replacementProof) + assert.match(workflow, /recreate-instances[\s\S]*?--instances "\$\{legacy_c3_instance\}"/) + assert.match(workflow, /incarnation_args=\(--previous-incarnation-digest/) + assert.match(workflow, /trap restore_legacy_c3_fallback EXIT/) + assert.match(workflow, /--mode restore-fallback[\s\S]*--general-cell-ids staging-gce-c2/) +}) diff --git a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs new file mode 100644 index 00000000000..f25e442d8c0 --- /dev/null +++ b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs @@ -0,0 +1,157 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + hasTerraformRoot, + renderAttributeConditions +} from './render-workload-identity-conditions.mjs' + +// GCP rejects an attribute_condition longer than this. +const ATTRIBUTE_CONDITION_LIMIT = 4096 + +const EXPECTED_CONDITIONS = { + staging: { + relay: { + github_staging_relay_capacity: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/prove-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/recover-relay-staging-c4-image.yml@refs/heads/main')) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-recover-relay-staging-c4-image.yml@refs/heads/main')))", + github_staging_relay_deploy: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-staging-gce-candidate.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/power-relay-staging.yml@refs/heads/main')) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-power-relay-staging.yml@refs/heads/main')))", + github_relay_asia_topology: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-asia-topology.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'))", + github_relay_asia_proof: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/prove-relay-asia-staging.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-asia-staging.yml@refs/heads/main'))", + }, + // The relay root creates this provider only in production, so staging has exactly one + // definition and it lives here. + apps: { + github: + "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-auth-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/load-skill-finalization-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/recover-skill-object-staging.yml@refs/heads/main')", + }, + }, + production: { + relay: { + github: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap.yml@refs/heads/main')))) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))))", + github_monitor: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/monitor-relay-production-job.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'))", + github_fence: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-multi-target.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-multi-target.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main'))", + github_production_relay_capacity: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-capacity.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-capacity-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap-job.yml@refs/heads/main'))) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main'))))", + github_relay_asia_topology: + "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-asia-topology.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'))", + }, + apps: { + github_production_app_deploy: + "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-auth-production.yml@refs/heads/main')", + }, + }, +} + +// Every repository the relay root accepts while the public extraction runs, with the workflow-ref +// head each one contributes. The apps root is not part of the dual accept. +const ACCEPTED_REPOSITORIES = [ + { + claims: + "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420'", + workflowHead: 'stablyai/orca-cloud/.github/workflows/' + }, + { + claims: + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420'", + workflowHead: 'stablyai/orca/.github/workflows/cloud-' + } +] + +// [root, provider, condition] for every provider the environment creates, across all roots. +async function flatten(environment) { + const rendered = await renderAttributeConditions(environment) + return Object.entries(rendered).flatMap(([root, providers]) => + Object.entries(providers).map(([provider, condition]) => [root, provider, condition]) + ) +} + +// Only roots whose directory ships can be rendered; the apps root stays in the private +// repository, so its expectations sit above unused until that directory is present. +const expectedRoots = (environment) => + Object.fromEntries( + Object.entries(EXPECTED_CONDITIONS[environment]).filter(([root]) => hasTerraformRoot(root)) + ) + +for (const environment of Object.keys(EXPECTED_CONDITIONS)) { + const roots = expectedRoots(environment) + test(`${environment} renders the exact reviewed attribute conditions`, async () => { + const rendered = await renderAttributeConditions(environment) + assert.deepEqual(Object.keys(rendered).sort(), Object.keys(roots).sort()) + for (const [root, providers] of Object.entries(roots)) { + assert.deepEqual(Object.keys(rendered[root]).sort(), Object.keys(providers).sort(), root) + for (const [provider, condition] of Object.entries(providers)) { + assert.equal(rendered[root][provider], condition, `${environment} ${root} ${provider}`) + } + } + }) + + test(`${environment} attribute conditions stay under the GCP length limit`, async () => { + for (const [root, provider, condition] of await flatten(environment)) { + assert.ok( + condition.length < ATTRIBUTE_CONDITION_LIMIT, + `${environment} ${root} ${provider} is ${condition.length} chars` + ) + } + }) + + test(`${environment} pins repository, branch, and environment on every provider`, async () => { + for (const [root, provider, condition] of await flatten(environment)) { + for (const pin of [ + "assertion.repository == 'stablyai/orca-cloud'", + "assertion.repository_id == '1273841466'", + "assertion.repository_owner_id == '127256420'", + "assertion.ref == 'refs/heads/main'", + `assertion.environment == '${environment}'` + ]) { + assert.ok(condition.includes(pin), `${environment} ${root} ${provider} is missing ${pin}`) + } + assert.ok( + condition.includes('assertion.workflow_ref ==') || + condition.includes('assertion.job_workflow_ref =='), + `${environment} ${root} ${provider} names no workflow` + ) + } + }) + + // A prefix or suffix match would turn each allowlist into a namespace grant. + test(`${environment} attribute conditions compare workflows only by equality`, async () => { + for (const [root, provider, condition] of await flatten(environment)) { + assert.doesNotMatch( + condition, + /startsWith|endsWith|matches|in \[/, + `${environment} ${root} ${provider}` + ) + } + }) +} + +// Why: the dual accept is only safe if each OR arm carries its own repository claims. An arm that +// inherited them, or a workflow ref that named the other repository, would let one repository's +// workflows run under the other's proof. +for (const environment of Object.keys(EXPECTED_CONDITIONS)) { + test(`${environment} admits both repositories through every relay provider`, async () => { + const rendered = await renderAttributeConditions(environment) + for (const [provider, condition] of Object.entries(rendered.relay)) { + assert.ok( + condition.startsWith("assertion.ref == 'refs/heads/main' && "), + `${provider} does not lead with the repository-independent claims` + ) + const refs = [...condition.matchAll(/(?:job_)?workflow_ref == '([^']+)'/g)].map( + (match) => match[1] + ) + const perRepository = ACCEPTED_REPOSITORIES.map((repository) => { + assert.ok(condition.includes(`(${repository.claims} && `), `${provider} misses an arm`) + return refs.filter((ref) => ref.startsWith(repository.workflowHead)).length + }) + assert.equal(refs.length, perRepository[0] + perRepository[1], `${provider} names a stray ref`) + assert.equal(perRepository[0], perRepository[1], `${provider} arms are not the same size`) + assert.ok(perRepository[0] > 0, `${provider} names no workflow`) + } + }) +} diff --git a/cloud/docs/orca-relay-capacity-testing.md b/cloud/docs/orca-relay-capacity-testing.md new file mode 100644 index 00000000000..73c0e5c6e5f --- /dev/null +++ b/cloud/docs/orca-relay-capacity-testing.md @@ -0,0 +1,259 @@ +# Orca Relay capacity testing + +This harness covers two different launch gates. The deterministic model proves that phase spreading produces the required aggregate heartbeat and auth-refresh rates without a synchronized cliff. The control harness opens real WebSockets, completes the host-key challenge, answers heartbeats, refreshes authorization, and reconnects with full jitter. + +Neither mode sends phone payloads or terminal content. Reports contain aggregate counts only. Access tokens and signing keys must be supplied through the documented secret paths and are never printed by the harness. + +## Deterministic 4k/10k model + +Run: + +```sh +pnpm load:relay:model +``` + +The default profiles are 4,000 and 10,000 standing controls over 15 modeled minutes. The command fails unless: + +- 15-second heartbeats produce approximately 267 and 667 pings per second; +- uniformly distributed 180–240-second token refreshes produce approximately 19 and 48 exchanges per second; +- one-second heartbeat and refresh bins remain inside the reviewed burst bounds. + +This is a schedule gate, not evidence that a cell can hold those connections. + +## Real control load + +Use a relay-scoped token source in one of two ways: + +1. Set `ORCA_RELAY_LOAD_ACCESS_TOKEN` and pass `--auth-origin`. Every control and refresh then uses the real auth-plane relay-token exchange. +2. Pass `--signing-key-file` containing the environment's auth signing key. This operator-only mode isolates relay capacity from auth capacity. Prefer memory-backed process substitution directly with `node`; `pnpm` may close that file descriptor. + +Never put either credential in command-line arguments, URLs, shell history, or reports. + +For staging, keep the signing key process-local and out of the filesystem: + +```sh +node dev/scripts/load-relay-controls.mjs \ + --director-origin https://relay-staging.onorca.dev \ + --auth-origin https://auth-staging.onorca.dev \ + --signing-key-file <(gcloud secrets versions access latest \ + --secret=orca-cloud-auth-signing-key \ + --project=onorca-cloud-staging) \ + --controls 840 \ + --ramp-seconds 210 \ + --duration-seconds 900 +``` + +Against a local or legacy combined service only: + +```sh +pnpm load:relay:controls -- \ + --target-origin http://127.0.0.1:8080 \ + --auth-origin http://127.0.0.1:8081 \ + --signing-key-file "$KEY_FILE" \ + --controls 800 \ + --ramp-seconds 210 \ + --duration-seconds 900 +``` + +Against the stable director and stamped cells: + +```sh +ORCA_RELAY_LOAD_ACCESS_TOKEN="$ACCESS_TOKEN" pnpm load:relay:controls -- \ + --director-origin https://relay-staging.onorca.dev \ + --auth-origin https://auth-staging.onorca.dev \ + --controls 800 \ + --ramp-seconds 210 \ + --duration-seconds 900 +``` + +The director form performs a real assignment for every generated relayHostId and follows the returned cell URL/epoch. Stamped GCE cells require this durable assignment; `--target-origin` cannot bypass it. Fresh identities must not exceed `hardCap - controlRebindReserve - unobservedBound` for the target cell. At the reviewed 1,000/60 policy, that placement ceiling is 840 even though the cell can hold 900 already-assigned ordinary controls. + +## Sharding + +Run one process per shard when the client machine becomes the bottleneck. Every shard receives a disjoint host/user index space: + +```sh +pnpm load:relay:controls -- \ + --director-origin https://relay-staging.onorca.dev \ + --auth-origin https://auth-staging.onorca.dev \ + --controls 1000 \ + --shard-count 4 \ + --shard-index 0 +``` + +Start shard indexes `0..3` with the same count and timing. `--controls` is per shard. Compare the combined client reports with Cloud Monitoring rather than summing only successful connection messages. + +## Required observations + +Capture these before and throughout a launch-gate run: + +- active controls, total connections, pending and active splices; +- auth successes/failures and refresh arrival rate; +- reconnect arrival distribution and close codes; +- SQL query failures and maximum interval latency; +- process queued bytes, heap, and event-loop p99; +- GCE instance CPU/memory, LB request/backend latency and 5xx metrics, plus Cloud Run director request/concurrency metrics; +- Cloud SQL CPU, connections, locks, and failover state. + +The real harness fails when fewer than 95% of requested controls become active, fewer than 95% remain active after ramp-up, or any steady-state connection, excess ramp retry, excess unexpected close, protocol, refresh, or socket error occurs. Public-load tests may set explicit small ramp-retry and close budgets; both default to zero and remain visible in the aggregate report. Use `--ramp-start-delay-ms` to desynchronize independent client shards. Failure reasons are reported only as bounded aggregate categories. `--allow-partial` exists only for diagnosing a known capacity boundary; a run using it cannot satisfy a launch gate. + +For hard-capped candidates, separately verify director placement stops at +`hardCap - controlRebindReserve - unobservedBound` and target ordinary socket +admission stops at `hardCap - controlRebindReserve`. Then overlap up to 100 +same-host replacement sockets that present valid authorization, assignment, +generation, and resume data and receive an encrypted host challenge, without +admitting unrelated controls into that reserve. The boundary mode proves +pre-activation socket headroom; activation and generation replacement remain +separate protocol tests. Above 100 concurrent replacements, verify the deployed +clients use the recorded bounded retry path. + +Use two independent staging proofs. The temporary 1,000/0 policy permits 900 +fresh assignments and exposes the exact physical boundary: + +```sh +node dev/scripts/load-relay-controls.mjs \ + --director-origin https://relay-staging.onorca.dev \ + --auth-origin https://auth-staging.onorca.dev \ + --signing-key-file <(gcloud secrets versions access latest \ + --secret=orca-cloud-auth-signing-key \ + --project=onorca-cloud-staging) \ + --controls 900 \ + --rebind-probes 100 \ + --rebind-hold-ms 4000 \ + --ramp-seconds 210 \ + --duration-seconds 900 +``` + +The command must keep all 100 replacements open for the complete hold, require +the next physical socket to receive HTTP 503, and pass the 15-minute soak. + +Then apply the final 1,000/60 policy and start a separate 840-control process. +During its explicit delay, rerun the reviewed 1,000/60 transition so the no-op +cell plan performs one exact C3 restart. The boundary checks begin only after +all 840 controls recover: + +```sh +CAPACITY_SA="$(gh variable get STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT --env staging)" +ORCA_RELAY_ADMIN_ID_TOKEN="$(gcloud auth print-identity-token \ + --impersonate-service-account="${CAPACITY_SA}" \ + --project=onorca-cloud-staging \ + --include-email \ + --audiences=https://relay-staging.onorca.dev/v1/admin/drain)" \ +node dev/scripts/load-relay-controls.mjs \ + --director-origin https://relay-staging.onorca.dev \ + --auth-origin https://auth-staging.onorca.dev \ + --signing-key-file <(gcloud secrets versions access latest \ + --secret=orca-cloud-auth-signing-key \ + --project=onorca-cloud-staging) \ + --controls 840 \ + --placement-overflow-probes 1 \ + --capacity-cell-id staging-gce-c3 \ + --capacity-hard-cap 1000 \ + --capacity-unobserved-bound 60 \ + --rebind-probes 100 \ + --allow-planned-transition-retries \ + --skip-rebind-overflow-check \ + --rebind-delay-seconds 1800 \ + --rebind-hold-ms 4000 \ + --ramp-seconds 210 \ + --duration-seconds 900 +``` + +The inline environment assignment keeps the one-hour admin token process-local; do +not export, print, or persist it. Before probing, the harness requires two advancing director +heartbeats after recovery with exact 840-connection telemetry, no pending connection +reservations, and the reviewed 1,000/60 policy. This run requires the 841st fresh +placement to receive the capacity-specific HTTP 503 response, then requires a newer +exact director heartbeat while all 840 ordinary controls remain unchanged. This proves at +least 100 replacement sockets remain available, and completes the normal soak. +The explicit planned-transition flag permits only connection retries before the +recovery gate. Expected drain closes and transition retries remain separately +counted; any missing recovery or steady-state error fails the proof. Never +persist generated host keys or resume credentials. + +## Cap transition and rollback order + +Changing the reviewed cap is a fail-closed operation. The capacity workflow +first builds and prunes a capacity-protocol-2 director rollback pair. It drains +C3, validates the saved Terraform plan, updates the director inventory, and +requires the old cell heartbeat to become stale. It then validates and applies +only C3's instance-template replacement and MIG pointer. A fresh heartbeat must +report the exact cap and unobserved bound before C3 returns to general +placement. Ordinary director deploys do not prune historical revisions. + +The staging proof uses reversible admission states. C3 moves to +`migration-only` and drains before replacement. After its fresh heartbeat, C3 +becomes the sole general cell while C2 moves temporarily to `migration-only`, +so fresh load identities can only land on C3. The restore mode first makes C2 +general as a safe fallback, verifies that C3 is healthy and non-draining, then +restores the reviewed C2/C3 general set. It never uses irreversible +`existing-only` for temporary isolation. + +The cell rollout is the forward point of no return. To restore 600, keep the +new director code active, configure 600 there first, then roll the empty cell +back to 600 and require its fresh matching heartbeat. Only after that may an +older director image be considered. Never shift director traffic to a +pre-protocol-2 image while any cell is configured above 600. + +## Soak profiles + +For the accelerated three-cycle gate, use a test deployment whose lease/rotation intervals are explicitly shortened together and run at both modeled 4k and 10k aggregate levels. Keep enough shards/cells that no client or cell exceeds its reviewed ceiling. Record listener, timer, socket, queued-byte, heap, SQL, and event-loop bounds at every cycle. + +For each real-time load-balancer canary, use exact GCE cell hosts through the public external HTTPS LB with the production 86,400-second backend timeout: + +- run a low-scale socket past the actual 86,400-second cap to prove the observed LB termination and configured-director recovery path; +- run a separate production-jitter canary spanning at least three proactive rotations before that cap; +- force at least three fixed-one GCE instance terminations and record recovery through strictly newer director assignments. + +The control harness is one input to those canaries. The full canary must also run real phone/data pairs and verify earlier-leg deadline handling, close/1006/502/503/504 recovery, subscription replay, deterministic rejection of ordinary in-flight RPCs, and idempotent credential reconciliation. These long-running canaries are public-launch gates; modest served load remains sufficient for implementation iteration. + +## Legacy recovery-wave gate + +After an isolated local or staging exercise, evaluate its aggregate report: + +```sh +pnpm load:relay:recovery-gate -- --report "$AGGREGATE_REPORT" +``` + +The evaluator has no HTTP or GCP client and rejects production project IDs, +production origins, unknown fields, malformed values, and reports larger than +1 MiB. It exits nonzero unless the report proves all numeric gates: + +- 760–840 draining desktops under 10,450–11,550 background requests/minute, + including the observed assignment-heavy mix and 8,500–10,500 assignment + `503` responses/minute; +- two targets, each capped at and observed below 600 connections, with every + desktop recovered; the report must separate physical sockets, in-flight + upgrades, pending host-data units, and their enforced sum; +- concurrent boundary probes proving every accepted phone can consume its + reserved host-data leg, control rebind remains available, established + sockets stay open, and failed upgrades release capacity exactly once; +- a 2-vCPU database, pool size 3, two total public slots, and one + resolve-priority slot; +- the production one-to-two-instance director range, Cloud Run concurrency 80, + and an observed two-instance peak during the wave; +- a separate old/new-revision rollout-overlap result that reaches four + processes and eight public operations while preserving the same resolve, + readiness, pool-wait, and database-CPU gates; +- zero migration expirations, migration aborts, retry exhaustion, readiness + failures, and non-public maintenance failures; +- one key-proven target registration per desktop, at least ten minutes on the + oldest migration lease at drain, and full-wave registration within five + minutes; +- at least 90% of baseline assignment throughput, at least 95% eligible + resolve success, and below 1% resolve overload; +- pool-wait p95 below 500 ms, pool-wait max below 5 seconds, database CPU p95 + below 70%, and database CPU max below 85%; and +- recovery within 14 minutes, one minute before the migration lease expires. + +The report must come from a controlled production-shaped run that joins client +counts with relay metrics and Cloud SQL monitoring. The gate validates evidence; +it does not generate the reconnect wave or prove that supplied observations are +authentic. Never use a hand-authored passing report as launch evidence. + +The rollout-overlap fields may come from a separate isolated sub-run. A +steady-state two-instance wave cannot substitute for the four-process overlap +created while old and new Cloud Run revisions coexist. The old side must run +the current production-equivalent shared gate with zero resolve-priority +slots; the new side must run the candidate two-public-slot scheduler with one +resolve-priority slot. diff --git a/cloud/docs/orca-relay-operations.md b/cloud/docs/orca-relay-operations.md new file mode 100644 index 00000000000..0f8eb7a50fb --- /dev/null +++ b/cloud/docs/orca-relay-operations.md @@ -0,0 +1,481 @@ +# Orca Relay operations runbook + +This runbook applies to the stable Cloud Run director and the production-shaped GCE cells in both environments. It does not authorize a full Terraform apply: staging and production contain unrelated drift, so inspect a saved targeted plan and its destroy count before every apply. + +The relay is automatically active for entitled signed-in desktops. There is no rollout flag, cohort, or user toggle. The emergency product kill switch is the auth plane refusing relay-token exchange; use cell drains only to move or terminate existing data-plane work. + +## Safety rules + +- Never put relay JWTs, access tokens, invite/resume credentials, or signing keys in URLs, shell history, logs, or reports. +- Mint the admin identity token with the exact configured audience, which is the stable director origin plus `/v1/admin/drain`, even when calling another admin path. +- A drain response contains only `recovery: resolve-director`. Never provide a recovery URL from a cell. +- Start the target revision and commit its assignment before asking the source control to drain. Keep the source revision alive while the desktop registers the verified target; existing source splices, installs, and confirmations stay origin-owned until their leases settle or grace ends. +- Treat `4401`, `4404`, and `4429` as endpoint-scoped. `4409`, `4503`, transport failure, `1006`, `503`, and `504` recover through the configured director. +- Resume recovery uses bounded `POST /v1/resolve`; an unexpired invite may use only the director compatibility WebSocket. +- Public assignment and resume recovery share two public database slots. A bounded + resolve waiter takes the next slot ahead of new assignments, leaving the third + director-pool connection for non-public work. +- Stop if a plan or command includes an unrelated service, database, bucket, destroy, or a `REPLACE_*` value. + +Set environment-specific values without printing the resulting token: + +```sh +export PROJECT_ID=onorca-cloud-staging +export REGION=us-central1 +export DIRECTOR_ORIGIN=https://relay-staging.onorca.dev +export DEPLOY_SERVICE_ACCOUNT=orca-cloud-staging-gha-deploy@onorca-cloud-staging.iam.gserviceaccount.com +export ADMIN_AUDIENCE="${DIRECTOR_ORIGIN}/v1/admin/drain" +ADMIN_TOKEN="$(gcloud auth print-identity-token \ + --impersonate-service-account="${DEPLOY_SERVICE_ACCOUNT}" \ + --audiences="${ADMIN_AUDIENCE}")" +``` + +Unset `ADMIN_TOKEN` when the operation finishes. + +## Preflight and stop conditions + +Before any rebalance, evacuation, deploy, or game day: + +1. Confirm `/health` on the stable director and every native target service URL. +2. Confirm the target cell is enabled, has a fresh heartbeat, and has reservation headroom for source units plus its migration lease. +3. Check active controls, splices, pending splices, queued bytes, auth failures, reconnects, SQL failures/latency, heap, and event-loop delay. +4. Confirm Cloud SQL is healthy and the auth JWKS endpoint is serving the expected current and rotation keys. +5. Start a timestamped operator log containing only resource names, epochs, aggregate counts, and response codes. + +Stop and roll back or leave the source intact if the target cannot register, source-owned operations do not drain, SQL errors rise, queue/heap alerts fire, or reconnect arrivals form a cliff. + +## Staging sleep and wake + +The `Power Relay Staging` workflow may stop staging outside internal test windows; it never targets +the production project. Its nightly 09:00 UTC sleep is guarded and fails safely when any cell +reports an active lease, request unit, migration, or observed connection. A successful sleep +disables cell admission, proves quiescence again, makes the active auth/director Cloud Run revisions +scale-to-zero compatible, scales every staging MIG to zero, and finally stops Cloud SQL. +After selector generation 1, scheduled sleep fails closed because waking would +require prohibited re-enablement. Selector-era wake restores every retained +cell process and verifies admission without rewriting it. + +Before staging testing, dispatch `wake` with the default `configured` selection. It starts Cloud +SQL, c1, and c2, waits for health, readiness, and authenticated heartbeats, and restores the +Terraform-declared admission cells; disabled c3 stays off. Before any staging Terraform apply or +candidate workflow, wake `all` cells. The local apply command deliberately rejects a stopped or +partially awake staging topology. + +Use `status` for a read-only view of SQL activation, MIG target sizes, and the minimum-instance +setting of the active Cloud Run revisions. If a sleep fails after admission was disabled, the +workflow attempts to restore previously enabled cells and leaves SQL and MIGs running. If status +shows SQL stopped while a MIG is nonzero, do not retry sleep: wake staging and inspect the partial +state first. + +## Legacy staging tagged-revision deployment + +The blue/green script remains for historical protocol fixtures, but live staging now uses GCE cells. This procedure is staging-only and must not be used to replace a production GCE cell. For each stamped cell the script: + +1. Adds a drain tag to the exact current 100%-traffic revision and queries its actual tag URL. +2. Registers the candidate cell as disabled, deploys its tagged revision with no traffic and the tag as its cryptographically bound public origin, queries that tag URL, and passes `/health`. +3. Clones the old image into a no-traffic previous-tag keeper with the old cell ID and its tag as the bound public origin. This is the safe return target for recently dormant assignments; the exact original revision remains the source of live controls. +4. Atomically records the keeper URL while disabling new source assignments, enables the candidate, starts bounded aggregate evacuation batches, and drains the exact original revision so desktops resolve the committed target. +5. Waits for every migration lease to report a key-proven target registration, shifts service traffic only after the verified count matches, then completes migrations only as source activity reaches zero. + +The workflow obtains Google identity tokens in memory and emits aggregate counts only. It retains drain/previous tags and disabled source-cell rows so live origin-owned work can finish and recently dormant assignments remain routable until normal dormant-TTL reassignment. Do not delete old tags while any assignment still resolves to their cell IDs. + +If the workflow stops before source disable, leave the disabled no-traffic candidate in place for inspection. If it stops after migrations begin, do not edit epochs or reservations and do not blindly re-enable the source. Follow the rollback rules below; a migration that never registered a target automatically returns to the source after its lease expires with another newer epoch. + +## Production GCE candidate deployment + +Production publishes an immutable relay image first, then declares a distinct disabled candidate cell in a reviewed targeted Terraform change. The candidate must have a new cell ID, exact hostname, route, backend service, fixed-one `RECREATE` MIG, and durable incarnation. Never replace the backend behind an existing cell origin. + +Run `Deploy Relay Production Candidate` in `preflight` mode before any mutation. The workflow verifies exact-host TLS, `/health`, dependency-backed `/ready`, a fresh authenticated heartbeat, the runtime service account, served digest, fixed-one topology, private-only networking, and survivor request-unit headroom. All production cells remain disabled until an explicit go-live decision. + +After launch, `execute` additionally requires the exact `EVACUATE` confirmation. It enables the proven candidate, disables new source assignment, and performs bounded target-first evacuation. Preserve the source route, backend, and MIG until every source-owned splice, install, confirmation, assignment, reservation, and migration lease is drained. Remove those resources only in a later reviewed Terraform change. + +## Production multi-target evacuation + +Use `Deploy Relay Production Multi-Target` when one candidate cannot hold the +source below the reviewed connection ceiling. Targets are sorted by cell ID, +and active assignments are apportioned deterministically before any mutation. +The workflow rejects a plan when projected or observed target connections +reach 600, request-unit reservations do not fit, or target topology and served +digests do not match Terraform. Current targets expose counts through their +authenticated runtime status. A legacy source without that field must have a +runtime-metrics log no older than 90 seconds; missing or stale telemetry is a +hard stop. + +Preflight separately proves that every source assignment can reserve one target +control. It subtracts enforced units and pending reservations from each target's +normal-admission pause instead of using the physical hard cap. Missing, stale, or +internally inconsistent connection-capacity telemetry is a hard stop. + +Only new-image cells explicitly configured with +`ORCA_RELAY_CELL_CONNECTION_HARD_CAP=600` and +`ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=` use the hard +cap. Configure the same values in the director's cell inventory. Existing +cells without both values retain their current 900-unit admission behavior. +Never add the limit to an old-image cell. + +For a limited cell, authenticated status and heartbeat report physical +connections, in-flight upgrades, pending host-data units, and their enforced +sum. The director admits one new control only while: + +```text +enforced connection units + + durable pending-control leases + + configured unobserved-arrival bound + < 600 - 100 control-rebind reserve +``` + +Missing, stale, wrong-incarnation, mismatched-cap, or internally inconsistent +telemetry makes that cell ineligible. The unobserved bound is the isolated +test's maximum arrivals over one heartbeat plus reaction latency and maximum +pre-auth/in-flight units, with 20% safety margin. Record p99 separately as an +SLO, not the safety bound. + +The target reserves two units for each accepted phone: the phone socket and +its future host-data socket. The second leg transfers that reservation instead +of competing for capacity. Exactly 100 units are exclusive to same-host +control rotation/rebind overlap; first controls, phones, and unreserved data +sockets stop at the target's exact 500-unit ordinary limit and cannot borrow +them. The director stops placement earlier at +`500 - unobserved-arrival bound`. Authenticated runtime status publishes both +thresholds so rollout automation can gate their exact values. At the hard ceiling the +target returns HTTP 503 for new work and leaves established sockets open. A +503 causes clients to consult the configured director, but a healthy assignment +normally resolves to the same cell; it is backpressure, not an automatic +migration. Rebind rejection within the tested 100-concurrent replacement +bound, or a new-phone rejection rate above the load-proven threshold, stops +rollout; excess simultaneous rebinds use the tested bounded retry path. + +Run `preflight` first with every target admission-disabled. `execute` requires +`EVACUATE_MULTI`, disables the source, enables targets serially, publishes each +target's fixed quota serially, and then rechecks all targets. It refuses +`/drain` unless the oldest active migration has at least ten minutes remaining. + +Any `/drain` attempt is rollback-unsafe because a lost response cannot prove +the old source did not accept it. Before that attempt, the workflow may restore +source admission only when no target registered. After it, keep the source +disabled and use `recover-forward`; never restore its old assignment epochs. +If the durable attempt is still exactly `prepared`, recover-forward may acquire +the original send permit once and send its original trace and grace. A +`send-may-have-started` attempt without an application receipt remains +ambiguous and must freeze; it is never resent. +For immutable legacy c3, the director durably records the exact cell +incarnation before the workflow sends `{v:1,graceMs:120000}`. A lost response +is never retried before the grace deadline plus 30 seconds. If authenticated +legacy aggregate counts still show active controls/splices then, +`recover-forward` may durably record and send at most one +`{v:1,graceMs:0}` request. It does not add unsupported fields to c3. +Before recover-forward, run the 15-minute production monitor with its explicit +`recover-forward` migration policy and exact existing-only source cell. That +signed policy permits only already registered migrations from that source whose +target control is inactive, which the recovery drain is intended to reconnect. +Recovery registers a conservative capacity snapshot and catches up newly active +source assignments for at most five passes; it still requires exact zero before drain. +Ordinary execute evidence remains strict, and +blocked or expired/unregistered migrations, inactive target runtimes, selector +drift, stale telemetry, and all capacity and database thresholds still stop +both the gate and immediate live recheck. + +If the old source has zero controls, splices, pending splices, activity leases, +and reserved units but closing transports prevent completion, run +`fence-source` with `FENCE_SOURCE`. It additionally requires every migration to +have a key-proven active target and zero source activity. The workflow resizes +the fixed-one source MIG to zero, proves no instance remains and its heartbeat +is stale, records a short-lived exact-incarnation fence attestation, then +completes through the shared strict database guard. A new heartbeat invalidates +the attestation. +If the resize response or workflow is interrupted, rerun the same fence mode; +an already-zero source resumes only after the retained topology and database +guards pass. + +Terraform intentionally ignores only operational MIG `target_size` drift, so a +later apply cannot recreate a fenced source. All other MIG topology remains +managed and preflighted. Do not resize a relay MIG directly: the guarded +workflow is the only supported zero-size path. + +## Dormant return and dormant rebalance + +A normal assignment may move only after every activity count is zero and the bounded dormant TTL has elapsed. A returning desktop with a stale epoch resolves/assigns through the director and accepts only the newer authenticated epoch. + +For an explicitly selected dormant assignment: + +```sh +curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/rebalance-dormant" \ + --header "Authorization: Bearer ${ADMIN_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "{\"v\":1,\"userId\":\"${USER_ID}\",\"relayHostId\":\"${RELAY_HOST_ID}\",\"targetCellId\":\"${TARGET_CELL_ID}\"}" +``` + +`assignment_active` is a stop result, not permission to clear counters. Investigate leases and wait; do not manually rewrite assignment rows. + +Validate that source reservations fall, target reservations rise by exactly one pending control, and the next returning desktop registers the returned epoch on the target. + +## Cell admission selector + +The director has three durable admission states: + +- `existing-only` preserves current assignments and sticky resolution but receives no ordinary, dormant, or dead-cell placement. +- `migration-only` receives only explicitly published evacuation assignments. +- `general` receives ordinary placement and may also receive explicit evacuation assignments. + +Before the selector boundary, the legacy `enabled` admin field remains compatible: +`false` means `existing-only` and `true` means `general`. Use the explicit +`state` field to stage migration-only targets. Once selector generation 1 is +committed, direct cell-state and cell-config admission changes fail closed. +Every later change must use the selector CAS. + +Deploy the selector-compatible director before cutover. Blue/green deployment +creates a separate `selector-rollback` revision at minimum instances zero, +promotes a second compatible revision, and retains older revisions through the +stabilization window. Cutover removes every older revision and refuses to +proceed unless only the compatible active and rollback revisions remain. + +First inspect the current generation and exact membership: + +```sh +curl --fail-with-body --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/admission-selector/status" \ + --header "Authorization: Bearer ${ADMIN_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data '{"v":1}' +``` + +Apply one exact, complete partition of every configured cell: + +```sh +curl --fail-with-body --request POST \ + "${DIRECTOR_ORIGIN}/v1/admin/admission-selector/apply" \ + --header "Authorization: Bearer ${ADMIN_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data @selector-request.json +``` + +`selector-request.json` must contain `v:1`, a unique 8–128 character +`attemptId`, the inspected `expectedGeneration`, and `membership` arrays named +`existingOnly`, `migrationOnly`, and `general`. A cell must appear exactly +once. For generation zero it must also contain `expectedMembershipSha256`: the +lowercase SHA-256 of the inspected membership's canonical UTF-8 JSON, with keys +in `existingOnly`, `migrationOnly`, `general` order and each array sorted. Hash +the current inspected membership, not the desired membership. Prefer the +audited `cutover-admission` operation, which derives this fingerprint directly +from its inspection. The transaction updates the compatibility boolean and all +tri-state membership atomically. + +If the apply response is lost or ambiguous, do not create a new attempt. +Inspect status with the same `attemptId`. Continue only when it reports either +`committed` with the exact intended generation and membership or `unchanged` +with the exact previous generation and membership. `diverged` is a freeze and +review result. Reapplying the identical attempt is idempotent. + +After the boundary is active, register new empty migration targets only through +`POST /v1/admin/admission-selector/add-migration-cells`. The request contains +one durable attempt ID, the exact current generation, and the complete new cell +configs, including canonical origins and reviewed connection limits. The +operation requires every cell and origin to be new, inserts inventory, +admission, and limit rows, extends migration-only membership, and advances the +selector exactly once in one transaction. Replay the same request after an +ambiguous response; changing its config or reusing its attempt ID fails closed. +One cell may be registered alone; evacuation and supersession still require +their reviewed multi-cell target sets. + +Use `retire-migration-cell` to stop exactly one migration-only cell from +receiving new migrations before fencing it. Supply only that cell as the target, +an exact durable selector attempt ID, and `RETIRE_MIGRATION_CELL`. The mode +requires the cell to be migration-only and applies one generation-bound CAS +that moves it to existing-only while preserving every other membership. + +For production, publish and blue/green deploy the selector-version-2 director +first. Add the disabled fixed-one Terraform cells, review a targeted plan with +zero destroys and replacements, and apply only their templates, MIGs, backend +services, and exact URL-map routes. Then run `Deploy Relay Production +Multi-Target` in `add-migration-cells` mode with `ADD_MIGRATION_CELLS`, a stable +attempt ID, and the reviewed unobserved bound. Wait for exact TLS, health, +readiness, digest, topology, and heartbeat evidence before including the new +generation in a preflight or monitoring gate. + +After generation 1, an `existing-only` cell can never return to +`migration-only` or `general`. A proven migration-only cell may transition to +general as additive capacity through a later CAS. Compatible director +restarts verify this boundary and do not restore legacy admission. + +Production cutover uses `Deploy Relay Production Multi-Target` in +`cutover-admission` mode with `CUTOVER_SELECTOR`. Supply the complete proven +general pool and migration-only target set. Every other Terraform cell becomes +existing-only in the single CAS. A lost response is inspected by attempt ID; +only the exact committed result is accepted. + +Also supply the exact `unobserved-connection-bound` produced by the passing +incident load gate. Before pruning old director revisions, the workflow proves +every proposed general or migration-only cell is a healthy fixed-one Terraform +deployment serving its pinned digest with a fresh heartbeat. Each must expose +the integrated runtime connection-capacity contract: hard cap 600, control +rebind reserve 100, the supplied unobserved bound, and the corresponding normal +admission pause. Cutover also requires fewer than 45 pre-auth connections and +enough pause headroom for current enforced units plus the director's durable +pending control reservations. + +The cutover workflow depends on the hard-cap release exposing +`connectionCapacity` from cell runtime status and +`pendingControlReservations` from director cell status. Missing or mismatched +evidence is a hard stop; do not weaken the gate to deploy the selector branch +alone. + +After cutover, candidate and multi-target workflows require existing-only +sources and migration-only targets. Pre-drain failure preserves that selector +membership and never restores legacy general admission. + +## Hot-cell rebalance + +1. Stop sending new assignments to a demonstrably unhealthy cell without changing already-issued client URLs: + +```sh +curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-state" \ + --header "Authorization: Bearer ${ADMIN_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "{\"v\":1,\"cellId\":\"${SOURCE_CELL_ID}\",\"enabled\":false}" +``` + +The disabled state survives director restart. Before selector generation 1, +explicitly re-enable the cell through the same endpoint only after health and +capacity checks pass. After generation 1, use a reviewed selector CAS and +never re-enable a legacy existing-only cell. +2. Move fully dormant assignments first with `rebalance-dormant`. +3. Recompute source/target request-unit headroom. One control costs one unit; one splice costs two; invite/install/confirmation/migration leases also reserve their contract units. +4. Move active assignments one at a time or in a bounded batch using the target-first evacuation below. +5. Pause whenever target total connections approach 800, queued bytes exceed 48 MiB, or any runtime/SQL alert fires. + +Do not infer required cells from DAU or phones. Use measured standing controls, splice/reservation distribution, startup/login bursts, and explicit safety margin. + +## Target-first active evacuation + +Start a durable migration on the director: + +```sh +curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/evacuate" \ + --header "Authorization: Bearer ${ADMIN_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "{\"v\":1,\"userId\":\"${USER_ID}\",\"relayHostId\":\"${RELAY_HOST_ID}\",\"targetCellId\":\"${TARGET_CELL_ID}\"}" +``` + +Record the returned `sourceCellId`, `targetCellId`, and strictly newer `assignmentEpoch`. The transaction reserves the target before publishing the epoch. + +Ask the source control to resolve the committed assignment while the source revision remains alive: + +```sh +curl --fail-with-body --request POST "${SOURCE_NATIVE_OR_TAG_ORIGIN}/v1/admin/drain" \ + --header "Authorization: Bearer ${ADMIN_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data '{"v":1,"graceMs":120000}' +``` + +Wait for the desktop to register a key-proven target control. Do not proceed on the basis of a socket open alone. Confirm the migration's target registration and that source-owned splices/install/confirmation work remains on the source control. + +Complete each migration only after source activity reaches zero: + +```sh +curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/migration-complete" \ + --header "Authorization: Bearer ${ADMIN_TOKEN}" \ + --header 'Content-Type: application/json' \ + --data "{\"v\":1,\"userId\":\"${USER_ID}\",\"relayHostId\":\"${RELAY_HOST_ID}\",\"assignmentEpoch\":${ASSIGNMENT_EPOCH}}" +``` + +`migration_target_not_registered`, `migration_source_still_active`, and `migration_target_not_active` are safety stops. Do not bypass them. + +### Dead-source completion + +Use only `Deploy Relay Production Multi-Target` in `fence-source` mode. The +workflow is the authority that proves the source MIG is durably zero, no +instance remains, retained topology matches Terraform, and the exact +incarnation heartbeat is stale before creating the short-lived database +attestation. There is no per-assignment dead-source operator endpoint. + +The aggregate operation is idempotent. A missing or expired attestation, new +source heartbeat, source activity, stale target heartbeat, inactive target +control, changed admission, request-unit drift, or aggregate reservation +mismatch is a stop result. + +### Registered-target supersession + +Use `Deploy Relay Production Multi-Target` in `supersede-target` mode with +`SUPERSEDE_TARGET`. The workflow requires the original source to remain +disabled, proves the failed target's retained topology, disabled admission, +MIG desired size zero, zero instances, and stale exact-incarnation heartbeat, +then records a short-lived fence attestation. It proves replacement health and +conservative connection headroom before enabling the replacement and invoking +the aggregate database operation. Rerun the same mode after a lost resize or +workflow response; it resumes from durable GCE and database state. + +Target supersession runs only through the IAM-authenticated private broker. The +requester cannot access Terraform state, Compute mutations, or director +mutation routes. The broker permits one configured cell triple, binds its +gitless checkout to the immutable image commit, and holds a durable GCS lease +through the exact-plan operation. It retains that lease after a failure so only +the same request can resume before expiry. This repair does not consume or +weaken the normal 15-minute source-recovery gate; run that unchanged gate only +after failed-target contention is gone. + +The transaction removes only the proven-dead target's leases, preserves +original-source activity, reserves replacement capacity, publishes exactly +`assignmentEpoch + 1`, and retires the old migration atomically. A repeated +identical request returns the same successor. A different concurrent +replacement, live current target, unavailable replacement, unexpected lease +topology, or accounting mismatch fails closed. + +## Dead cell + +Do not call an unreachable cell or trust it to supply a target. For each affected assignment: + +1. Verify another cell has capacity. +2. Start the director evacuation to publish a newer target epoch. +3. Let desktop `1006`/transport recovery resolve the configured director and register the target. +4. Phones resolve through the director; unexpired invites use the director compatibility route. +5. Complete only after expired source activity leases release and the target control is active. + +If the dead cell returns, leave its stale epoch fenced. Never decrement an epoch or restore an old assignment row. + +## Global admission or memory pressure + +At connection headroom, queue, heap, or event-loop alerts: + +1. Preserve established controls/splices and reject new pre-auth work through the existing admission reserves. +2. Identify whether pressure is source-local, cell-wide, auth/JWKS, SQL, or a synchronized reconnect event using aggregate metrics only. +3. Move dormant work first. Evacuate active work only when a target has measured request and memory headroom. +4. If every cell is unsafe, make the auth plane refuse new relay-token exchanges, then issue authenticated drains with full-jitter client recovery. This is the kill switch; do not add a flag. + +Never raise runtime admission or queued-byte limits during an incident without a load result proving memory headroom. + +## Drain and SIGTERM + +Planned drain uses the authenticated admin endpoint and a reviewed grace that permits target-first registration and origin-owned operations to settle. Phones and desktops always recover through the configured director. + +SIGTERM is a degraded path. It may advertise zero grace, rejects new work, closes phone/data pairs together with `4503`, and cannot guarantee target registration first. Game-day this separately from planned drain. + +After either path, verify close/`1006`/`504` observations, bounded reconnect arrival, subscription replay, deterministic ordinary in-flight rejection, and idempotent install/confirmation reconciliation. + +## Rollback + +Before target registration, an expired migration automatically releases target reservations and returns the assignment to the source with another strictly newer epoch. Verify: + +- the migration is marked aborted; +- target pending-control and migration leases are gone; +- source reservation is restored exactly once; +- resolved epoch is newer than both the original and failed target epochs. + +Once a target control is registered, do not force the pre-registration rollback. Keep the source control and operations alive, repair the target, or perform a new target-first evacuation to a healthy cell. Never edit epochs or reservation counters manually. + +After a deployment traffic shift, preserve the old revision/tag until metrics and live reconnect checks pass. If the new revision is unhealthy, shift traffic back only while old controls are still valid, then issue a strictly newer director migration rather than reusing a prior epoch. + +## Game-day matrix + +Run and record each scenario in staging before launch: + +- kill a cell instance mid-session and observe configured-director recovery; +- send zero-grace SIGTERM and compare it with authenticated drain; +- hold an invite install and a resume confirmation across drain; +- wedge a slow receiver until the bounded queue closes only that splice; +- delay mobile Blob conversion and confirm text/binary counter order; +- rotate JWKS while controls refresh; +- make auth unavailable through expiry plus 60-second existing-splice-only grace, then recover with distributed jitter; +- fail SQL during install/confirm and verify only `not-found` or the one committed result is externally visible; +- return a dormant host, overload a cell, kill a cell, evacuate active work, and exercise pre-registration rollback. + +The served black-box relay suite validates the protocol/state transitions used by these procedures. The physical-device and real-GFE canaries remain separate launch gates; unit/black-box success cannot replace them. diff --git a/cloud/docs/relay-incident-monitor.md b/cloud/docs/relay-incident-monitor.md new file mode 100644 index 00000000000..3e5fc04836e --- /dev/null +++ b/cloud/docs/relay-incident-monitor.md @@ -0,0 +1,152 @@ +# Relay incident monitor + +Status: monitor core and dedicated identity boundaries are implemented locally; +the targeted production bootstrap and live negative-permission proof remain +required before dispatch. + +This monitor is read-only. It probes Relay endpoints and reads Cloud Monitoring, Compute +inventory, and authenticated aggregate director status. It does not drain, restart, resize, +deploy, change admission, or write to Google Cloud. + +## Production workflow + +Run `Monitor Relay Production` manually. Choose: + +- `dry-run` for the required 15-minute pre-drain gate. +- `monitor` for a 90-minute incident watch. + +Use the default `strict` migration policy for ordinary mutations. Select +`recover-forward` only after a drain attempt has durably registered migrations +and enter its exact existing-only recovery source cell. Recovery evidence tolerates +only the aggregate count of registered migrations whose target control is not +currently active for that source. Blocked or expired/unregistered migrations and every other +health threshold remain unchanged. + +Enter the exact selector generation and all three disjoint membership sets: +`existing-only`, `migration-only`, and `general`. Every configured cell must +appear exactly once. Generation zero is the only mode that reads the legacy +boolean admission field; selector-era generations use only the durable +tri-state membership. Any generation or membership mismatch freezes the gate. +The workflow verifies dependencies and restored evidence before +authentication. It has no shared-deploy fallback and accepts only the dedicated +monitor provider and service-account production environment variables. Do not +dispatch it until the targeted identity bootstrap is applied and its live +negative-permission checks pass. + +The job polls every 60 seconds and writes private aggregate evidence at 0, 5, 15, 30, 45, 60, 75, +and 90 minutes where applicable. It uploads state JSON, checkpoint JSONL, and Markdown for 14 +days. No tokens, request bodies, logs, user IDs, host IDs, or relay device IDs are recorded. +Reruns keep one stable incident ID, restore the immediately preceding private +artifact, verify its commit/run/attempt provenance and content hashes, and pass +`--restart`. A missing or mismatched artifact fails closed. A missing, stale, +or collector-failed sample is durably recorded and resets the active continuous +window. The next fresh sample starts a new 15- or 90-minute window under the +same incident lineage. + +Exit code `2` means the gate froze or a dry run failed. Missing, stale, malformed, unauthorized, or +unavailable telemetry fails closed. + +## Local use + +The active `gcloud` identity must be a service account that can mint an ID token for the exact +audience `https://relay.onorca.dev/v1/admin/drain`, and it needs read access to the monitored GCP +resources. A user account normally needs Token Creator on an approved service account. + +For a short run, an already minted JWT may instead be supplied through +`ORCA_RELAY_ADMIN_ID_TOKEN`. The token must remain valid for the whole run and is never persisted. +Use refreshable service-account credentials for the 90-minute mode. + +```sh +pnpm incident:relay -- \ + --environment production \ + --incident-id relay-incident-20260728 \ + --expected-selector-generation 2 \ + --expected-existing-only-cells production-gce-c1 \ + --expected-migration-only-cells production-gce-c2 \ + --expected-general-cells production-gce-c3,production-gce-c4,production-gce-c5,production-gce-c6 +``` + +Add `--pre-drain-dry-run --duration-minutes 15` for an ordinary gate. Add +`--migration-policy recover-forward --recovery-source-cell-id ` only +for a committed forward-recovery gate. Durable files default to +`.relay-incidents/`. If the process stops, rerun the identical command with `--restart`. A runner +gap resets the active window at the next fresh sample and preserves the prior +window evidence. A threshold freeze never clears automatically. + +A production candidate or multi-target mutation must download the exact +dry-run artifact by workflow run ID and attempt. It verifies the artifact +hashes and provenance, requires a green completed 15-minute state no older +than five minutes, then rechecks the live selector and one complete fresh +sample of every safety signal immediately before running the mutation command. +The signed state binds `strict` evidence to ordinary mutations and +`recover-forward` evidence to the exact recover-forward source; neither can +authorize the other. +The monitor and mutation jobs share one production lock. A passing dry-run is +durably marked consumed before mutation and cannot authorize another run. + +## Freeze thresholds + +| Signal | Freeze condition | +| --- | ---: | +| Active probe age | over 60 seconds | +| Cloud/log data age | over 180 seconds | +| Cell heartbeat age | over 45 seconds | +| Endpoint latency | over 2,000 ms | +| Cloud SQL CPU | over 80% | +| Cloud SQL memory | over 90% | +| Cloud SQL backends | over 250 (62% of the verified 400-connection ceiling) | +| Cloud SQL waiting backends | over 20 | +| Cloud SQL deadlocks | over 0 | +| Relay pool waiters | over 800 | +| Relay pool wait | over 2,500 ms | +| PostgreSQL retries in five minutes | over 300 | +| Exhausted PostgreSQL retries | over 0 | +| Director instances | outside 5–6 | +| Director CPU or memory | over 80% | +| Director concurrency | over 64 | +| Unexpected director 5xx or auth 5xx in five minutes | over 0 | +| Connections per cell process | over 500 | +| Queued bytes per cell process | over 48 MiB | +| Blocked or expired/unregistered migration | over 0 | +| Registered migration with inactive target | over 0, except in `recover-forward` evidence | + +Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh +heartbeats, and matching live admission. + +## Implementation log + +- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to + 800 waiters / 2,500 ms (2026-08-27). Basis, measured from + `orca_relay_runtime_metrics` (`databasePoolWaitersMax`, + `databasePoolWaitMsMax`): healthy fleet-wide bursts reach 43 waiters and + 2.03 s several times an hour (52 burst-minutes over three days), a cell + roll's reconnect surge peaks at 676 waiters, and the 2026-08-23 incident + peaked at 356 waiters without ever crossing 2.5 s — amplitude does not + separate incident from routine operation in either direction, and a + 15-minute gate had roughly one-in-six odds of freezing on a burst. The + retry signals discriminate that incident at ~10x separation and keep their + thresholds; the pool bars now fence only unbounded queueing. +- Recalibrated the Cloud SQL backends freeze from 160 to 250 (2026-08-26). + Basis, measured from `cloudsql.googleapis.com/database/postgresql/num_backends` + latest-sum over 24 healthy hours: mean ~100, 1-minute spikes to 216, with + 10 minutes over the old bar of 160 — enough to freeze roughly one in ten + 15-minute pre-drain gates on baseline noise. 250 clears measured healthy + peaks and still fires well before the verified 400-connection ceiling; + pool waiters, pool wait latency, and exhausted retries keep their strict + thresholds. +- Recalibrated the PostgreSQL-retry freeze from 20 to 300 per five minutes + (2026-08-26). Basis, measured from + `jsonPayload.event="orca_relay_postgres_transaction_retry"` in production + logs: healthy-day bursts reach 234/5min with zero exhausted retries and 26% + of five-minute windows over 20, while the 2026-08-23 lock-contention + incident ran roughly 2,200–3,000/5min. Exhausted retries stay at zero + tolerance. +- Added a fail-closed state machine with latched threshold freezes, + generation-scoped checkpoint boundaries, continuity-reset evidence, cadence + accounting, restart-gap recovery, and the 15-minute pre-drain gate. +- Added Cloud Monitoring, active-probe, relay runtime, and authenticated director collectors. +- Added exact Cloud SQL instance and Cloud Run service filters, five-minute + DELTA aggregation, and serialized aggregate admin reads so monitoring cannot + load the director's three-connection database pool. +- Added private atomic state, idempotent JSONL checkpoints, and secret-safe Markdown evidence. +- Added the manual production workflow. It has not been dispatched. diff --git a/cloud/docs/relay-terraform-fencing-plan.md b/cloud/docs/relay-terraform-fencing-plan.md new file mode 100644 index 00000000000..7d8664f19f5 --- /dev/null +++ b/cloud/docs/relay-terraform-fencing-plan.md @@ -0,0 +1,149 @@ +# Relay Terraform Fencing Recovery Plan + +Status: private broker implementation in review; no production mutation performed + +## Current status + +- `2d7dc31` commits Terraform-owned per-cell zero/one desired size and removes + the lifecycle ignore. +- The follow-up slice implements private exact saved plans, SHA-256 binding, + durable attempt evidence, lost-response inspection, resume, proven + pre-apply abort, and exact-incarnation attestation. +- Focused Terraform-fence, multi-target, relay database, assignment-store, and + black-box tests pass locally. +- Production remains untouched. A rollout still requires review, CI, the + director/schema deployment, an initial output-state refresh, and a separate + reviewed fence-set commit for any cell selected during an incident. +- The private Cloud Run broker owns the exact Terraform checkout, saved plans, + state access, Compute mutation, and a durable GCS mutation lease. The + workflow requester can invoke the broker and read aggregate evidence but + cannot mutate Terraform state, Compute, or director fence routes. + +## Goal + +Make production relay-cell fencing a Terraform-managed, resumable operation. +The workflow must never attest a cell as fenced until Terraform state, GCE, +runtime identity, and retained routing all prove the same exact cell +incarnation is offline. + +## Safety boundary + +- Do not deploy, mutate GCP, push, or create a pull request during implementation. +- Keep the cell route and backend while its MIG is fenced at size zero. +- Treat any Terraform apply whose start cannot be disproved as recover-forward. +- Never restore a fenced cell automatically after an apply may have started. +- Store plans in a private temporary directory and remove them on every exit. +- Never print, upload, or commit complete plan JSON. +- Fence modes are fail-closed until a private mutation broker owns the narrow + state/plan and exact-cell Compute permissions. The exact-workflow GHA fence + account must never receive those permissions or director mutation routes + directly; it has only aggregate reads and fence-attempt status. +- The monitor identity has no Terraform state access. It can call only the + aggregate selector, cell, evacuation, and runtime status routes. +- The broker accepts only its configured source, failed target, and + replacement target. Its image commit must equal the reviewed fence commit; + callers cannot override topology, Terraform paths, commands, or identities. + +## State model + +1. Add `relay_gce_fenced_cells` as the committed set of cell IDs whose desired + MIG size is zero. +2. Derive every cell's desired size from that set: fenced is zero, otherwise + one. +3. Reject unknown fenced IDs and any topology outside the zero-or-one + invariant. +4. Remove the MIG `target_size` lifecycle ignore so Terraform owns the fence. +5. Persist one durable attempt record keyed by an unguessable attempt ID. It + binds environment, cell ID, exact cell incarnation, MIG/generation + identity, fence commit, saved-plan digest, GCE operation, creation time, + expiry, and terminal status. + +## Operation sequence + +### 1. Prepare and guard + +1. Require the explicit fence confirmation and a clean checkout at the exact + fence commit. +2. Confirm the committed production fence set contains the requested cell. +3. Read Terraform state and live topology, then bind the attempt to the exact + MIG, instance group, backend, origin, and cell incarnation. +4. Run the existing admission, assignment, lease, migration, connection, + heartbeat, backend, and route guards before planning. + +### 2. Create and validate the exact plan + +1. Create a private temporary directory with mode `0700` under `umask 077`. +2. Write the saved plan there and compute its SHA-256 digest. +3. Inspect only narrow fields from `terraform show -json`. +4. Require exactly one relevant in-place MIG update from target size one to + zero. +5. Reject create, delete, replace, unrelated update, route/backend removal, or + any different cell action. +6. Persist the pending attempt evidence before apply. + +### 3. Apply or recover forward + +1. Recheck the pre-apply guards and saved-plan digest immediately before apply. +2. Apply the exact saved plan with Terraform locking. +3. If apply fails or its response is lost, inspect Terraform state, live MIG + size, remaining instances, and the relevant GCE operation. +4. If apply may have started, retain the committed fence set and keep polling + forward until the MIG converges to zero or a bounded, diagnosable failure is + recorded. +5. Resume idempotently from durable evidence after workflow or runner loss. + +### 4. Abort before apply + +1. Allow abort cleanup only when evidence proves apply never began. +2. Require Terraform state and live MIG to remain at one with the original + identity and topology. +3. Mark the attempt aborted, remove the cell from the committed fence set + through a separately reviewed commit, and delete local plan artifacts. +4. If apply start is ambiguous, refuse abort and recover forward. + +### 5. Attest + +1. Require Terraform state target size zero. +2. Require live MIG target size zero and no managed instances. +3. Require the exact MIG/generation identity and retained route/backend + topology to match the attempt. +4. Require admission disabled and the exact runtime heartbeat stale. +5. Require unexpired durable attempt evidence and the same saved-plan digest, + fence commit, and GCE operation. +6. Record the exact-incarnation cell fence and complete the attempt in one + database transaction. + +## Implementation slices + +- Terraform: variable, per-cell desired size, validation, outputs, tfvars, IAM. +- Relay contract: durable attempt schema, store methods, admin endpoints, exact + attestation binding, retention/expiry. +- Deployment tooling: private plan lifecycle, digest and selector validation, + apply/recovery/abort state machine, GCE operation inspection. +- Workflow: explicit prepare/apply/resume/abort modes and no direct MIG resize. +- Runbooks: committed fence-set, exact-plan, recover-forward, and reviewed + abort procedures. + +## Required tests + +- Normal fence plan and apply. +- Lost or ambiguous apply response recovers forward. +- Resume when Terraform state and live MIG are already zero. +- Pre-apply guard failure performs no mutation. +- Proven pre-apply abort permits committed fence-set cleanup. +- Ambiguous apply refuses abort cleanup. +- No attestation before every Terraform, GCE, route/backend, heartbeat, and + exact-incarnation condition passes. +- Saved-plan validation rejects unrelated, replacement, create, and destroy + actions. +- Attempt evidence rejects wrong environment, cell, incarnation, MIG, + generation, commit, digest, operation, expiry, and terminal state. + +## Validation + +- Focused Node and relay contract tests. +- `pnpm test`, `pnpm typecheck`, and `pnpm lint`. +- Sequential relay contract and relay builds after schema/API changes. +- Terraform 1.15.8 `fmt -check -recursive`, `init -backend=false`, and + `validate`. +- `git diff --check`, local commit, and clean worktree. diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md new file mode 100644 index 00000000000..14bb2a25d7c --- /dev/null +++ b/cloud/docs/relay-workflows.md @@ -0,0 +1,402 @@ +# Relay GitHub Actions Configuration + +The `cloud-*` workflows in `.github/workflows/` are the Relay deploy and +operate surface. Every one of them is gated on the repository variable +`ORCA_CLOUD_OPERATIONS_ENABLED == 'true'` and does nothing until the repository +owner sets it. The app and auth deploy workflows this document once also +covered stay in the private `stablyai/orca-cloud` repository. + +Set these staging environment variables before running the staging deploy workflow: + +```text +STAGING_GCP_REGION +STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER +STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT +STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER +STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT +STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER +STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT +STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER +STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT +``` + +`STAGING_GCP_REGION` exists today only as a repository variable. Create it as a +staging **environment** variable before deleting any repository-level variable; +`Deploy Relay Staging` gates its whole job on it being non-empty, so a +delete-before-create silently skips it. + +The Relay deploy, capacity, and Asia values come from the matching staging +Terraform outputs after the targeted identity bootstrap: + +```sh +terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_service_account + +gh variable set STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' +gh variable set STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT --env staging --body '' + +terraform -chdir=infra/terraform output -raw github_staging_relay_capacity_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_staging_relay_capacity_service_account + +gh variable set STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' +gh variable set STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT --env staging --body '' +terraform -chdir=infra/terraform output -raw github_relay_asia_proof_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_relay_asia_proof_service_account +gh variable set STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' +gh variable set STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT --env staging --body '' +``` + +The capacity provider accepts only this repository's capacity proof and +bootstrap workflows on `main` with the `staging` environment. It does not fall +back to the shared deploy identity. + +The Relay deploy provider accepts exactly five workflows on `main` with the +`staging` environment: Bootstrap Relay Staging Capacity, Deploy Relay Staging, +Deploy Relay Staging GCE Candidate, Operate Relay Asia Admission, and Power +Relay Staging. Set both `STAGING_GCP_RELAY_DEPLOY_*` variables before merging +the workflow repoint; the job gates read them and skip while they are unset. + +Set these separately before enabling production deploys: + +```text +PRODUCTION_GCP_REGION +PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER +PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT +PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER +PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT +PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER +PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT +PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER +PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT +PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER +PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT +PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT +PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT +``` + +The Relay operations values come from matching Terraform outputs. Set them as +production GitHub environment variables, not repository fallbacks. Every one of +them is relay-owned in `infra/terraform`: + +```sh +terraform -chdir=infra/terraform output -raw github_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_deploy_service_account +terraform -chdir=infra/terraform output -raw github_relay_monitor_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_relay_monitor_service_account +terraform -chdir=infra/terraform output -raw github_relay_fence_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_relay_fence_service_account +terraform -chdir=infra/terraform output -raw github_production_relay_capacity_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_production_relay_capacity_service_account +terraform -chdir=infra/terraform output -raw relay_director_runtime_service_account +terraform -chdir=infra/terraform output -raw relay_runtime_service_account + +gh variable set PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT --env production --body '' +gh variable set PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT --env production --body '' +``` + +Run those commands only from the audited operator session after the targeted +identity bootstrap apply. The providers require their exact workflows on +`refs/heads/main` with the `production` environment. Missing values fail +closed; no dedicated operations identity falls back to the shared deploy identity. + +The shared production identity is restricted to seven named direct Relay callers plus the exact +regional-rehome and same-cap reusable wrapper/job pairs on `main` in the `production` environment. +Its Artifact Registry and Cloud Run mutation permissions are scoped to the Orca repository, Relay +director, and Relay fence broker; it cannot mutate the API or auth services. + +Bootstrap the production capacity identity only after its reviewed commit is on +`main`. Reinitialize the production backend explicitly, save the exact targeted +plan, require **9 additions, 0 changes, and 0 deletions**, then apply that saved +plan. `manage_artifact_dns=false` keeps the unimported Cloudflare records out of +this GCP-only operation. + +```sh +export GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)" +terraform -chdir=infra/terraform init -reconfigure \ + -backend-config=backend/production.hcl -input=false +terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \ + -var-file=environments/production.tfvars -var manage_artifact_dns=false \ + -target=google_iam_workload_identity_pool_provider.github_production_relay_capacity \ + -target=google_service_account.github_production_relay_capacity \ + -target=google_service_account_iam_member.github_production_relay_capacity_workload_identity_user \ + -target=google_project_iam_custom_role.github_production_relay_capacity_mutation \ + -target=google_project_iam_member.github_production_relay_capacity_mutation \ + -target=google_project_iam_member.github_production_relay_capacity_viewer \ + -target=google_project_iam_member.github_production_relay_capacity_artifact_reader \ + -target=google_storage_bucket_iam_member.github_production_relay_capacity_state \ + -target=google_service_account_iam_member.github_production_relay_capacity_runtime_user \ + -out=/tmp/orca-relay-production-capacity-identity.tfplan +terraform -chdir=infra/terraform show /tmp/orca-relay-production-capacity-identity.tfplan +terraform -chdir=infra/terraform apply /tmp/orca-relay-production-capacity-identity.tfplan +unlink /tmp/orca-relay-production-capacity-identity.tfplan +``` + +Confirm a second targeted plan is empty before setting the two production +environment variables from reviewed Terraform outputs. + +`Deploy Relay Asia Topology` is the only workflow allowed to add the reviewed +`asia-east2` network and fixed-one cell topology. Its dedicated identity is +bound to that exact workflow, `main`, `workflow_dispatch`, and the selected +GitHub environment. The workflow always saves a targeted plan, rejects any +delete, replacement, US-resource, SQL, DNS, certificate, or unrelated change, +and applies only the exact validated plan. It always passes +`manage_artifact_dns=false`; observability and IAM are separate targeted +operations. + +Before the first admission operation, publish and deploy a compatible director image while the +topology remains unchanged. Verify the exact serving digest, health, readiness, and rollback tag; +older directors reject the generation-zero membership fingerprint. After a topology apply, use +`Operate Relay Asia Admission` in `inspect` mode to read the exact live selector generation. If and +only if it is generation 0, run +the explicit `initialize` mode with the exact membership SHA-256 printed by +`inspect` and `INITIALIZE_ADMISSION_SELECTOR`; the director checks both under its +database lock, so this freezes the existing membership without adding, removing, +or moving a cell and rejects intervening drift. Then +atomically register the new cells as migration-only, binding every mutation to +the exact live selector generation and a durable attempt ID. Deploy and verify +the director configuration only after registration, then promote C27 alone before C28/C29. +Rollback returns +Asia cells to migration-only; it does not destroy the network or use +existing-only. The production topology dispatch remains unavailable until the +published compatible image is committed for C27-C29. + +`Prove Relay Asia Staging` runs from a dedicated ephemeral repository runner in +`asia-east2` with the `relay-asia-east2-load` label. It promotes only staging C4, runs four bounded +load shards at the exact 3,000/6,000 shape, validates continuous cell/director/Cloud SQL evidence, +and always returns C4 to migration-only before publishing evidence. Register the runner with +`--ephemeral` immediately before dispatch so it accepts one proof job and then removes itself. Each +shard exchanges its +exact workflow OIDC identity for a ten-minute in-memory staging token; no load +credential, signing key, or raw load output is stored or uploaded. + +Production promotion evidence must prove the exact production manifest, not an independent rebuild. +Before refreshing C4, target and apply only +`google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer` +from the staging state with `manage_artifact_dns=false`. Run `Publish Relay Production Image` in +`mirror-staging` mode with the exact digest and typed confirmation, then run `Deploy Relay Staging` +with that digest. The mirror validates identical source and target manifest digests; the staging +deploy binds the request to C4's checked-in image and deploys the director by that same digest. +Only then refresh empty migration-only C4 and run the proof. The proof and production promotion both +reject a serving director whose runtime digest differs from the cell/evidence digest. + +Expected project IDs: + +```text +staging: onorca-cloud-staging +production: onorca-cloud +``` + +Production relay delivery keeps the stable director separate from GCE cell rollout. `Publish Relay +Production Image` builds and prints an immutable digest. `Deploy Relay Production Director` accepts +only that digest, performs a health-gated Cloud Run director update, and never deploys data-plane +cell stamps. Its explicitly confirmed prune option retains only the serving and cold rollback pair, +and runs only after both compatible revisions pass the capacity-protocol health gate. Use that gate +before adding Asia cells so an incompatible dormant revision cannot be routed later. A reviewed +Terraform candidate pins the same digest on a distinct disabled GCE cell; +`Deploy Relay Production Candidate` then runs read-only preflight or an explicitly confirmed +target-first evacuation. Staging uses the same GCE data-plane shape as production; `Deploy Relay +Staging GCE Candidate` exercises the reviewed GCE preflight and evacuation state machine before +production use. + +`Prove Relay Staging Capacity` is the only cap-transition path. Apply mode +reversibly moves `staging-gce-c3` to migration-only, drains it, validates the +saved director and C3 Terraform plans, updates the director first, and requires +stale telemetry before replacing the exact C3 template and MIG. A fresh +matching heartbeat is required before C3 becomes the sole general placement +cell; C2 remains recoverable in migration-only. Restore mode does not depend on +Terraform or image agreement: it restores C2 first, then restores C3 only after +a fresh, healthy, non-draining capacity check. The same transition order +restores 600 before an older director image can be used. + +Its bounded C4 refresh mode keeps Asia admission migration-only, accepts only an exact predecessor +or already-applied target digest, validates a saved two-resource image-only plan, fences and proves +C4 empty before replacement, and requires an empty targeted readback afterward. It cannot change +C4 capacity, routing, trust configuration, or any production resource. + +`Recover Relay Staging C4 Image` runs independently after a failed, timed-out, or cancelled C4 +refresh and can also be dispatched with `RECOVER_STAGING_ASIA_C4_IMAGE`. It verifies the triggering +job and both exact Terraform end states, preserves a fully converged ready target or predecessor, +and fences partial state before restoring the pinned predecessor through an exact saved two-resource +plan. A separate no-credential supervisor requeues a recovery cancelled while waiting for the shared +staging mutation lane. Admin credentials are refreshed around Terraform. Before the first refresh, +target only +`google_iam_workload_identity_pool_provider.github_staging_relay_capacity`, require exactly one +in-place condition update, apply the saved plan, and require an empty targeted readback. + +This identity cannot bootstrap its own Relay authorization. Before the first +capacity dispatch, use the existing audited staging blue/green deploy path to +roll the compatible image and verified `ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT` +onto both director revisions. Roll C2/C3 through saved, validated cell plans +with `Bootstrap Relay Staging Capacity` while they remain at 600/60. That +workflow keeps the deploy identity only for Relay admin calls and uses the +capacity identity for Terraform and GCP mutations. It isolates, drains, rolls, +verifies, and restores one cell at a time, with the other cell as its failure +fallback. Commit the matching +C2/C3 image and capacity pairs in staging tfvars, then require the capacity workflow's read-only 600/60 +verification to pass. Only a later reviewed configuration commit may select +1,000/0 or 1,000/60. The capacity workflow carries the reviewed director +topology through blue/green; it never targets the drifted director or its Cloud +SQL dependencies with Terraform. + +The one-time bootstrap recognizes only the exact pre-capacity C2/C3 image and +its five-field runtime-status response. It first proves C2 as the general fallback, +then isolates and drains C3 and requires zero durable activity plus two fresh, +instance-bound zero runtime metrics. After restarting the fixed-one MIG, it +requires two new zero samples and a new director heartbeat incarnation started +after the restart before restoring C3. This clears the legacy process's +unreported drain flag without treating missing runtime fields as proof. Any +other image, response shape, activity, or stale evidence fails closed with C2 +preserved as the general fallback. Reruns classify partial C2/C3 progress. A +legacy target may carry only no capacity record or the exact stale 600/60 record +before the idempotent director update; afterward the exact stale record is +required until that cell is replaced. + +`Power Relay Staging` lowers the staging bill when no internal testing is underway. It runs a +guarded sleep attempt at 09:00 UTC every day and also supports manual `status`, `wake`, and `sleep` +dispatches. Manual mutations require the exact `WAKE_STAGING` or `SLEEP_STAGING` confirmation. +Sleep refuses to stop a cell with active Relay work, disables admission and checks again, then +scales the three GCE MIGs to zero and stops the shared staging Cloud SQL instance. Wake starts SQL, +waits for healthy workers and authenticated heartbeats, then restores only the admission state +declared in Terraform. The default wake starts c1/c2; choose `all` before a Terraform apply or GCE +candidate operation so the complete Terraform-owned topology is running. + +`Deploy Relay Production Multi-Target` handles a source that cannot fit on one +candidate. It serializes deterministic per-target quotas, enforces each +target's reviewed 600- or 1,000-connection gate and the ten-minute lease gate, +and treats a drain attempt as the +rollback point of no return. Fence and fence-abort remain fail-closed. +It also registers one additive migration cell and retires exactly one +migration-only cell through explicit, generation-bound selector operations. +Registered-target supersession invokes the IAM-only private broker, which owns +the durable mutation lease, exact Terraform checkout, saved plans, state, and +narrow Compute mutation. The workflow requester has read and broker-invocation +authority only; it never receives those mutation permissions directly. +`Deploy Relay Fence Broker` updates only that service's immutable image and +requires the digest to carry the exact `sha-${GITHUB_SHA}` tag. Terraform +continues to own its identity, scaling, IAM, environment, and deletion +protection. +Its `add-migration-cells` mode is the selector-safe path for newly provisioned +empty targets after generation 1. It requires `ADD_MIGRATION_CELLS` and a +stable selector attempt ID, but no pre-drain artifact because it moves no +assignments. Run the fresh 15-minute gate only after the new cells are +registered and healthy. + +## Cloud SQL rollout lease + +Every workflow that mints a Cloud Run revision or applies a relay instance template against a shared +Cloud SQL instance takes the compare-and-swap lease in `.github/actions/cloud-sql-rollout-lease` +immediately after `google-github-actions/setup-gcloud`. The per-repository `concurrency` groups +(`production-cloud-sql-rollout`, `relay-staging-mutation`) only serialize runs inside one repository; +once the relay workflows live in `stablyai/orca` there are two queues pointed at one instance, and +`relay-cloud-sql-connection-budget.mjs` computes `rolloutOverlap` as a `Math.max` that is only sound +with one rollout in flight. Keep both the groups and the lease. + +| Environment | Bucket | Object | +| ----------- | -------------------------------------- | --------------------------------------------------- | +| production | `onorca-cloud-terraform-state` | `terraform/state/cloud-sql-rollout/production.lock` | +| staging | `onorca-cloud-staging-terraform-state` | `terraform/state/cloud-sql-rollout/staging.lock` | + +`Deploy Relay Asia Topology` and `Operate Relay Asia Admission` pick the pair from +`inputs.environment`. `Deploy Relay Production Capacity` and `Deploy Relay Production Same-Cap` call +their reusable job several times per run, so every wave job acquires with `release: 'false'` under +the run-scoped default holder key and a single `if: always()` `release_lease` job frees it once every +wave has finished. + +`Monitor Relay Production` stays off the lease. It is read-only, holds only viewer roles, and putting +it on a durable lease would let monitoring block a rollout and a rollout block monitoring. +`dev/scripts/production-cloud-sql-rollout-lock.test.mjs` enforces the group, the lease wiring, and a +content-derived census of every rollout candidate against +`dev/scripts/cloud-sql-rollout-lock-census.mjs`. + +`Monitor Relay Production` is manual and read-only. Its `dry-run` mode enforces the 15-minute +pre-drain gate; `monitor` records a 90-minute incident watch. Both require the +operator to enter the exact selector generation and tri-state membership. The +workflow must use a dedicated identity for aggregate monitoring and +exact-audience read-only Relay-admin calls. Do not dispatch it until that +monitor identity, exact workflow-bound WIF trust, and read-only admin-route +authorization have been bootstrapped. +Capacity-transition monitoring binds the evidence to one exact general cell. It +still blocks all migration failures and any inactive registered migration from +that cell or another serving cell; it permits only inactive rows +from unrelated existing-only cells because a capacity restart neither creates +nor advances assignment migrations. +Reruns restore hash-verified private state from the prior attempt. Production +candidate and multi-target mutations require a fresh dry-run artifact and +recheck its exact selector and every live safety signal before any mutation +command. All three workflows share the production deployment lock, and each +passing dry-run artifact is marked consumed before the mutation starts. +The dry-run lineage fails closed after 25 total minutes, so continuity resets cannot extend the +15-minute gate indefinitely. +Missing or stale telemetry fails closed, and the workflow uploads only private aggregate +Markdown/JSON evidence. + +`Deploy Relay Production Capacity` is the only production cap-transition path. It runs only +from `main` and accepts exactly the current general rollout set: C7-C10, C13-C16, and C19-C26. +C17/C18 and every existing-only, draining, fenced, or disabled cell are excluded in code. Its +Terraform/GCE phase uses the dedicated exact-workflow capacity identity. Read-only checks, selector +isolation, drain, and the audited director blue/green update use the existing shared production +deploy identity; the production environment and common deployment lock still gate those steps. +Apply mode consumes a fresh 15-minute monitor gate bound to the selected cell, moves only that cell +from general to migration-only, drains it, updates only its director capacity entry, and applies a +saved validated plan for only its template and MIG. Previously completed 1,000 cells remain +unchanged while later 600 cells roll. The selected cell returns to general only after a fresh +matching 1,000/60 heartbeat. The restart gate waits up to 15 minutes for genuine activity to finish +while preserving every zero-work check. Rollback performs the same isolated sequence to 600/60 +without waiting on a cell that may already be unhealthy. If the selected cell cannot answer the +drain call, rollback instead requires two stale-heartbeat snapshots with zero durable activity +before replacing it. Its typed confirmation includes the exact selected cell so a form-selection +mistake cannot downgrade another cell. Interrupted Terraform applies resume only when the planned +current template has the exact reviewed image, capacity, and identity and the remaining change is +that selected MIG update or obsolete-template deletion. Production configuration pins only the +approved serving set to the compatible image and 1,000/60; the transition classifier accepts only +the reviewed mixed 600/1,000 envelope until every selected cell converges. Every GCP-only Terraform +command disables artifact DNS. Any failed mutation leaves only the selected cell migration-only and +never changes another cell's selector state. + +After multiple production cells pass the canary path, `wave-apply` may raise two to four reviewed +600/60 serving cells under one fresh 15-minute capacity-transition gate. The first cell is bound to +the sealed evidence; every later cell derives the exact expected selector generation and reruns the +complete live preflight before mutation. After the first cell, continuation preflights retry only +missing or stale signal evidence for at most one minute; health, threshold, selector, and migration +failures stop immediately. Cells still drain, restart, and verify sequentially. A +failed cell stays isolated and prevents every later wave job from starting; earlier completed cells +remain general at 1,000/60. The workflow lock, single-use evidence marker, exact predecessor check, +targeted Terraform plan, and per-cell heartbeat/admission oracle are unchanged. + +`Deploy Relay Production Same-Cap` rolls only the reviewed US 1,000/60 and Asia 3,000/60 serving +sets without changing a cell's connection shape. Use `canary-apply` for exactly one cell. A successful canary +seals its commit, target and rollback digests, selector generation, and durable rehome generation; +`batch-apply` accepts only that same authority and rolls two to four cells sequentially. Each cell is +isolated, drained to two restart-safe samples, replaced from a targeted saved plan, and restored only +after a new incarnation reports the exact digest, cap, heartbeat, and rehome protocol. The durable +worker must remain disabled throughout. The post-restart trust check is application-mediated by the +director; the workflow never receives or mints a director or stamped-cell runtime token. A failure +keeps only the selected cell migration-only, while the exact rollback digest remains dispatchable via +the same workflow's `rollback` mode. + +The first compatible director rollout uses `bootstrap-runtime-identity=true` with +`BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY`. That one-time path requires the exact stamped-cell +predecessor identity, creates both the cold rollback and candidate on the distinct director identity, +and proves the disabled durable control through those compatible revisions before moving traffic. +Later director deploys reject the predecessor identity and verify the disabled control on the serving, +rollback, and candidate revisions. + +`Operate Relay Production Rehome` is the only durable worker control. `inspect` is read-only; +`enable` is selector-, director-digest-, rollback-digest-, and control-generation-bound, starts at +exactly 10 hosts per minute, consumes the fresh 15-minute safety monitor, and seals 24 hourly buckets +of aggregate requested-region, selected-region, fallback, and unavailable-region evidence with +positive Asia requests and selections. `pause` and `disable` apply their generation CAS immediately +after checkout and authentication, before package installation, revision checks, or log diagnostics. +Their typed confirmations are `PAUSE_REGIONAL_REHOMING` and `DISABLE_REGIONAL_REHOMING`. Keep the +default 3,600,000 ms drain grace so existing splices can finish. The job summary contains only fresh +aggregate active, receipt, registration, completion, and abort counts. diff --git a/cloud/docs/staging-relay-deploy-identity-rollout.md b/cloud/docs/staging-relay-deploy-identity-rollout.md new file mode 100644 index 00000000000..81fd3efe44a --- /dev/null +++ b/cloud/docs/staging-relay-deploy-identity-rollout.md @@ -0,0 +1,177 @@ +# Staging Relay deploy identity rollout + +Moves the five staging Relay workflows off the apps-owned `github_deploy` +account and onto the relay-owned `github_staging_relay_deploy` account +(`orca-cloud-staging-gha-relay`). This is a **rollout**, not state surgery, so +it lives beside [`terraform-root-split-runbook.md`](./terraform-root-split-runbook.md) +rather than inside it: that runbook is state-only and runs no apply, and every +step below applies. + +Production is untouched. `local.relay_github_deploy_service_account_email` +still renders `orca-cloud-gha-deploy` there, and the production relay plan slice +is byte-identical to the pre-split single-root baseline. + +## What moves, and what it costs + +The staging Relay runtime allowlists exactly one deploy account +(`ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT`, `apps/relay/src/admin-token-verifier.ts`), +so the account, the director env, the four cell startup scripts, and the +workflow variables have to change together. Plan on a staging window in which +no Relay workflow runs. + +| Change | Cost | +| --- | --- | +| 10 new identity resources | Create only. No compute. | +| 6 relay bindings repoint to the new account | Delete + create. The old account loses them. | +| `ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT` on the director | New Cloud Run revision. | +| Cells c1–c4 startup metadata | Four instance-template replacements, one MIG repoint each. | + +## Preconditions + +- [ ] PR 13 is merged, so both accounts are declared and the census test passes. +- [ ] No staging Relay workflow is running or queued. All five share the + `relay-staging-mutation` concurrency group; `prove-relay-staging-capacity` + and `recover-relay-staging-c4-image` are in it too. +- [ ] Staging is awake, or you accept waking it as part of step (e). + +## (a) Compute-free identity apply + +Targeted apply on the staging relay root. Verified against a post-surgery state +copy: **10 creates, nothing else**. + +```sh +terraform -chdir=infra/terraform apply \ + -var-file=environments/staging.tfvars \ + -target='google_service_account.github_staging_relay_deploy[0]' \ + -target='google_iam_workload_identity_pool_provider.github_staging_relay_deploy[0]' \ + -target='google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user[0]' \ + -target='google_storage_bucket_iam_member.github_staging_relay_deploy_state_list[0]' \ + -target='google_storage_bucket_iam_member.github_staging_relay_deploy_state[0]' \ + -target='google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader[0]' \ + -target='google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer[0]' \ + -target='google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer[0]' \ + -target='google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user[0]' \ + -target='google_project_iam_member.github_staging_relay_deploy_compute_viewer[0]' +``` + +Reject the plan if it shows anything but those ten creates. + +## (b) Publish the two variables + +```sh +terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_workload_identity_provider +terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_service_account + +gh variable set STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' +gh variable set STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT --env staging --body '' +``` + +Nothing reads them until (c) merges, so this step is reversible on its own. + +## (c) Repoint the workflows and flip the relay bindings + +The workflow repoint ships in PR 13. Merging it and applying the six repointed +bindings is one step, because the new account cannot operate without them and +the old account must not keep them: + +```sh +terraform -chdir=infra/terraform apply \ + -var-file=environments/staging.tfvars \ + -target='google_project_iam_member.github_staging_relay_power[0]' \ + -target='google_service_account_iam_member.github_relay_runtime_service_account_user[0]' \ + -target='google_service_account_iam_member.github_relay_director_runtime_service_account_user[0]' \ + -target='google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor[0]' \ + -target='google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder[0]' \ + -target='google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer[0]' +``` + +Expect **six replacements plus one create**: the target on +`github_relay_director_runtime_service_account_user` drags +`google_service_account.relay_director_runtime`, which staging has never +created. That create is additive and does not move the director onto the new +identity; only applying `google_cloud_run_v2_service.relay` does that. Drop +that one `-target` if you would rather leave it to the staging drift +remediation, and accept that the new account then has no `serviceAccountUser` +on the director runtime account. + +The director's `ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT` changes only through +`google_cloud_run_v2_service.relay`, and applying that address in staging also +carries the whole staging director backlog: the identity swap to +`orca-cloud-staging-relay-dir`, `timeout` 3600s to 30s, concurrency 1000 to 80, +the four-cell topology, and roughly twenty new env entries. Do it as part of +the staging identity remediation in the drift plan, in this same window, with +that plan reviewed on its own terms. + +## (d) Roll the cells, one at a time + +Use **Prove Relay Staging Capacity**. It authenticates only as +`STAGING_GCP_RELAY_CAPACITY_*`, that is `github_staging_relay_capacity`, never +the new deploy account, and the capacity identity's admin routes +(`RELAY_CAPACITY_ADMIN_ROUTES`) cover every call the roll makes. It is +therefore unaffected by this cutover in either direction. + +Per cell the targeted apply is: + +```text +-target='google_compute_instance_template.relay_gce_cell["staging-gce-c"]' +-target='google_compute_instance_group_manager.relay_gce_cell["staging-gce-c"]' +``` + +That plan is one template replacement plus one MIG update, and it also drags an +in-place update of `google_compute_health_check.relay_gce_liveness[0]` from the +standing staging log_config drift. Confirm it is in place, not a replacement. + +Two gaps to plan around: + +- `prove-relay-staging-capacity` has arms for **c3 and c4** only. +- `bootstrap-relay-staging-capacity` rolls **c2 and c3**, but it mints its admin + token as the deploy account. Running it across the email change fails: it + verifies a rolled cell with a token that cell no longer allowlists. +- **c1 has no workflow roller.** Roll c1, and c2 if you do not use bootstrap, + by the same two-target apply under human review inside the window. + +Wait for each MIG to report stable before starting the next cell. + +## (e) Verify + +Dispatch **Power Relay Staging** in `status` mode. It exercises the new +credential end to end: Workload Identity exchange on the new provider, the +state read, `gcloud sql instances describe` and the MIG reads through +`orcaRelayStagingPower`, `run services describe` on both the director and the +shared staging auth service, and an admin `cell-status` call that only succeeds +if the director allowlists the new account. Then run a `sleep` and a `wake` to +exercise the mutation paths. + +## (f) Retire the staging Relay grants on the shared account + +Follow-up PR against `infra/terraform-apps`. Once (e) passes, the staging +`github_deploy` account no longer needs the Relay-only reach it has today. +Narrow, in the apps root: + +- `google_project_iam_member.github_compute_viewer` — kept only for the Relay + candidate preflight; no staging app workflow reads GCE topology. +- `google_project_iam_member.github_cloud_run_developer` — project-wide today; + the Relay services are now covered by the two service-scoped grants above, so + the apps root can scope it to the API and auth services. +- `google_project_iam_member.github_artifact_writer` — still needed by the app + deploys; verify before touching. + +Leave alone: the AR mirror writer +(`google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer`) +names the **production** deploy account by literal and is unrelated to this +change; `github_runtime_service_account_user` and +`github_auth_runtime_service_account_user` are still used by the staging app +deploys. + +## Rollback + +Before (c): revert the two variables, or unset them. The job gates skip while +they are empty, and the old account still holds every grant. + +After (c) but before the cells are rolled: re-apply the six bindings from the +previous commit, which points them back at `orca-cloud-staging-gha-deploy`, and +revert the workflow repoint. The new account and its provider can stay; they +grant nothing the old path needs. + +After the cells are rolled: roll forward. Rolling four templates back costs the +same as rolling them forward and leaves the same window. diff --git a/cloud/infra/terraform/.terraform.lock.hcl b/cloud/infra/terraform/.terraform.lock.hcl new file mode 100644 index 00000000000..13cb0e4644e --- /dev/null +++ b/cloud/infra/terraform/.terraform.lock.hcl @@ -0,0 +1,67 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/external" { + version = "2.4.0" + constraints = "~> 2.3" + hashes = [ + "h1:AmY6ZeIvqoTT5ZjzD+P49PeQH6Va1QLMkX+7MUQfYoA=", + "h1:gXyK3ZkweDqkwEV9waEDWpljUY4yZXi/nRUSEuJn83k=", + "zh:0772afb42b658468ac5e15df33bf2080456f8f0b8ab163bfe9c50d2b2ea02135", + "zh:0ac31a9aaa43dfcff5944b791596cdc94e153348e4bb4642282d034dff548134", + "zh:32d8492b1bdcc956ca3c6d00c6392d0a83942ff11d4820c7ee63ca6796e06950", + "zh:3c0482e894429f528ce6655a76ab0d8a9f7c0dacc6c828865e1515d4a7dbb852", + "zh:61e68100b4db2f930b31491f23c602126382fd5e51252be1b551f0e17f8ddbee", + "zh:6d60f615a0ad85eb962c9eb94f25e3eba7a72684ce276ba5dfb23f36b295a8f8", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9ced2745eb5f1346203027d2dd7bf856ad1d279a25730ff7dbc6eec187aaca0c", + "zh:a8378558a177d43f55aa0d79d4fae91a704695122a1b109668c1daa8fb76f09d", + "zh:aadd98086133d3ebea67437d56512fdcc6dfb3bd34dfc23f276c0db9272e27b4", + "zh:beff701b653841e70441978137768f54e7dc6c27e7bf12a4589087f01f5bbcee", + "zh:c91c2223b29fdbc0044d20e1936ccc051d010727a13f2ff1e75e51f09bff33a3", + "zh:d491f9c2d32a39dc4031628469ae7c8aec0074312a7c1f0286b173cdcf854a54", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "6.50.0" + constraints = "~> 6.0" + hashes = [ + "h1:79CwMTsp3Ud1nOl5hFS5mxQHyT0fGVye7pqpU0PPlHI=", + "h1:mhrmzHgoQoall8+7hA9Lpy0HAnjNC1N5+sPDp6bGizM=", + "zh:1f3513fcfcbf7ca53d667a168c5067a4dd91a4d4cccd19743e248ff31065503c", + "zh:3da7db8fc2c51a77dd958ea8baaa05c29cd7f829bd8941c26e2ea9cb3aadc1e5", + "zh:3e09ac3f6ca8111cbb659d38c251771829f4347ab159a12db195e211c76068bb", + "zh:7bb9e41c568df15ccf1a8946037355eefb4dfb4e35e3b190808bb7c4abae547d", + "zh:81e5d78bdec7778e6d67b5c3544777505db40a826b6eb5abe9b86d4ba396866b", + "zh:8d309d020fb321525883f5c4ea864df3d5942b6087f6656d6d8b3a1377f340fc", + "zh:93e112559655ab95a523193158f4a4ac0f2bfed7eeaa712010b85ebb551d5071", + "zh:d3efe589ffd625b300cef5917c4629513f77e3a7b111c9df65075f76a46a63c7", + "zh:d4a4d672bbef756a870d8f32b35925f8ce2ef4f6bbd5b71a3cb764f1b6c85421", + "zh:e13a86bca299ba8a118e80d5f84fbdd708fe600ecdceea1a13d4919c068379fe", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fec30c095647b583a246c39d557704947195a1b7d41f81e369ba377d997faef6", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.0" + constraints = "~> 3.6" + hashes = [ + "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", + "h1:lVDv+0AjDjrLfpmaJbWqUmIw/k3/AHXLc3N4m55SNdo=", + "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", + "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", + "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", + "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", + "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", + "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", + "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", + "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", + "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", + "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", + "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", + ] +} diff --git a/cloud/infra/terraform/README.md b/cloud/infra/terraform/README.md new file mode 100644 index 00000000000..2e2b9b02011 --- /dev/null +++ b/cloud/infra/terraform/README.md @@ -0,0 +1,409 @@ +# Terraform + +This root manages the Orca Cloud relay and nothing else. It requires Terraform >= 1.7 +(`removed` blocks); OpenTofu at that floor works too. + +## Three roots + +Orca Cloud is three Terraform roots sharing one project and one state bucket per environment, +with a different prefix each. They are separate so the relay can be extracted into a public +repository without carrying the app plane, its database passwords, or its Cloudflare credential +with it. + +| Root | Directory | State prefix | Owns | +| --- | --- | --- | --- | +| foundation | `infra/terraform-foundation` | `terraform/foundation` | Project service enablement, the Artifact Registry repository, the API runtime service account, the Cloud SQL instance, the GitHub Workload Identity pool, the Cloud SQL rollout lease grant | +| apps | `infra/terraform-apps` | `terraform/apps` | The API and auth services, the artifact and skill-package buckets, the skill plane and its observability, auth and artifact DNS, the app deploy identities | +| relay | `infra/terraform` | `terraform/state` | Everything relay: the director, GCE cells, the fence broker, relay observability, and the relay operator identities | + +Apply order on a greenfield project is **foundation first**, then relay and apps in either order. +The other two roots reach foundation only by literal or by `data` lookup, never through +`terraform_remote_state`: foundation state holds the Cloud SQL instance and apps state holds +generated database passwords in cleartext, and the relay root must not acquire a read path into +either once it is public. Each root's substitution for a foundation value is pinned by +`dev/scripts/terraform-root-partition.test.mjs`, which asserts every declared resource family is +owned by exactly one root per environment. + +Three families are owned per environment rather than outright: the shared deploy service account, +its Workload Identity provider, and its WIF binding live in the relay root for production and in +the apps root for staging, with complementary counts. Every IAM binding on that account follows +it. `dev/fixtures/terraform-root-partition/families.json` is the authority. + +### The carve is complete + +Both state surgeries have run (`docs/terraform-root-split-runbook.md`), so this root's state holds +only relay families and the `removed` guard blocks from the window are gone. The shared deploy +identity (`google_service_account.github_deploy`, its provider, and its bindings) is declared here +with production-only counts; staging's copies are declared by `infra/terraform-apps`. An untargeted +plan is orderable again; the `Plan:` line still reflects the standing cell-template drift backlog. + +### Dual-accept Workload Identity during the public extraction + +While the relay source moves to the public `stablyai/orca` repository, every relay Workload +Identity provider accepts the same workflows from both repositories. `github_accepted_repositories` +lists the extra repositories; `relay-github-workflow-trust.tf` renders one parenthesised OR arm per +accepted repository, each arm carrying that repository's own `repository`, `repository_id`, and +`repository_owner_id` claims plus its exact workflow refs. `ref`, `environment`, and `event_name` +stay outside the OR. Workflow files keep their names in the private repo and take the +`workflow_file_prefix` (`cloud-`) in the public one. + +Adding a repository is a tfvars edit: no provider block changes, and the rendered strings are +pinned by `dev/scripts/workload-identity-attribute-conditions.test.mjs`. An empty list renders +byte-identically to the single-repository form, which is what makes the arms reviewable against +the pre-extraction condition. + +Closing the cutover is an owner step, in this order: + +1. Retire the private workflows, so nothing runs from `stablyai/orca-cloud` any more. +2. Point `github_owner`, `github_repo`, `github_repo_id`, and `github_owner_id` at + `stablyai/orca` (`1183888342`, owner `127256420`), and set `workflow_file_prefix` for it by + moving the surviving entry's prefix onto the primary: the public files keep the `cloud-` names, + so the primary prefix becomes `cloud-` unless the files are renamed back. +3. Empty `github_accepted_repositories` in both `environments/*.tfvars`. +4. Re-render and update the pinned conditions, then apply. Each provider goes back to a single + arm, and `google_service_account_iam_member.github_accepted_repository_workload_identity_user` + is destroyed as the primary `attribute.repository` binding takes over. + +Step 2 and step 3 must land in the same apply: dropping the accepted entry before repointing the +primary would revoke the public repository mid-flight. + +### `ORCA_RELAY_IMAGE_DIGEST` is not Terraform-owned + +`deploy-relay-blue-green.mjs` sets `ORCA_RELAY_IMAGE_DIGEST` on the director container at deploy +time, but `relay.tf` does not declare it and the director's `ignore_changes` cannot name a single +list element. A director apply from this root therefore strips that variable. Terraform is not the +owner today: deploy through the director workflow, and treat any direct +`google_cloud_run_v2_service.relay` apply as something that needs the next deploy to restore the +digest. Giving Terraform the variable (a declared input the deploy script writes through) is +tracked as follow-up work in the split checklist, not in this change. + +Select a root with `--root`; omitting it keeps the relay root, so existing callers are unchanged. + +```sh +pnpm infra:init --env staging --root foundation +pnpm infra:plan --env staging --root apps +``` + +## Bootstrap Remote State + +The GCS backend bucket must exist before `init`. + +Staging: + +```sh +gcloud storage buckets create gs://onorca-cloud-staging-terraform-state --project onorca-cloud-staging --location us +gcloud storage buckets update gs://onorca-cloud-staging-terraform-state --versioning +``` + +Production: + +```sh +gcloud storage buckets create gs://onorca-cloud-terraform-state --project onorca-cloud --location us +gcloud storage buckets update gs://onorca-cloud-terraform-state --versioning +``` + +One bucket per environment holds all three roots' state under separate prefixes, so this is a +one-time step for the whole project. + +Do not commit `.tfstate`, `.tfplan`, or `.terraform` files. + +## Production app deploy identity: moved + +The production app deploy identity, the skill alert channel guard, and every +other app-plane resource now live in `infra/terraform-apps`. Their bootstrap +procedure moved with them; run it with `-chdir=infra/terraform-apps`. This root +no longer declares the API service, the auth service, the artifact or +skill-package buckets, the Cloud SQL databases, or the app DNS records, and it +no longer needs a Cloudflare or 1Password credential. + +## Staging Relay capacity identity bootstrap + +Before the capacity workflow can mutate staging, create a saved targeted plan +containing only `github_staging_relay_capacity` providers, accounts, roles, +bindings, and outputs. Apply it with backend locking, then require a targeted +refresh/no-op plan. The identity is bound to the exact workflow on `main` and +the `staging` environment. Its state write access is limited to the default +staging state and lock object prefix. + +Copy these outputs into same-named staging GitHub environment variables: + +1. `github_staging_relay_capacity_workload_identity_provider` +2. `github_staging_relay_capacity_service_account` + +Before dispatch, prove it can read the saved state and reviewed Relay +resources, but cannot change unrelated Cloud Run services, templates, managed +instance groups, databases, DNS, or secrets. + +The identity bootstrap alone does not authorize Relay admin routes. Publish the +compatible image, then use the existing staging blue/green deploy path to carry +and verify the capacity service account on both director revisions. Use saved, +validated cell plans through `Bootstrap Relay Staging Capacity` to roll C2/C3 +at 600/60. The reviewed staging tfvars must pin the same image and capacity. +Require a read-only 600/60 capacity-workflow +run before reviewing either 1,000-policy configuration. Do not target the +director with Terraform: its dependency closure includes unrelated live drift. + +## Production Relay incident identity bootstrap + +Before running the production monitor, create a saved targeted plan containing +only the two dedicated service accounts, their exact-workflow +providers/bindings, and monitor read roles. Reject any Cloud Run, GCE, +database, network, runtime-service-account, or unrelated IAM change. Apply +that plan with backend locking, then run a targeted refresh/no-op plan. + +Copy these outputs, in order, into same-named production GitHub environment +variables documented in `.github/workflows/README.md`: + +1. `github_relay_monitor_workload_identity_provider` +2. `github_relay_monitor_service_account` +3. `github_relay_fence_workload_identity_provider` +4. `github_relay_fence_service_account` + +Use an audited operator session for the GitHub variable writes. Before +dispatch, prove the monitor account can read required aggregate telemetry but +cannot mutate Relay or state. Fence modes remain disabled until a separate +private broker owns and validates the exact state, plan, cell, and durable +attempt boundary; never grant direct Compute update or Terraform-state write +access or director mutations to the GHA fence account. + +## Relay Asia topology identity bootstrap + +Bootstrap each environment's Asia topology identity with operator credentials +before dispatching its workflow. IAM cannot bootstrap itself. Reinitialize the +exact backend and save a targeted plan that +contains only these twelve additive resources: + +1. `google_iam_workload_identity_pool_provider.github_relay_asia_topology` +2. `google_service_account.github_relay_asia_topology` +3. `google_service_account_iam_member.github_relay_asia_topology_workload_identity_user` +4. `google_project_iam_custom_role.github_relay_asia_topology_mutation` +5. `google_project_iam_member.github_relay_asia_topology_mutation` +6. `google_project_iam_custom_role.github_relay_asia_topology_read` +7. `google_project_iam_member.github_relay_asia_topology_read` +8. `google_artifact_registry_repository_iam_member.github_relay_asia_topology_artifact_reader` +9. `google_storage_bucket_iam_member.github_relay_asia_topology_state` +10. `google_project_iam_custom_role.github_relay_asia_topology_state_list` +11. `google_storage_bucket_iam_member.github_relay_asia_topology_state_list` +12. `google_service_account_iam_member.github_relay_asia_topology_runtime_user` + +The state-list role contains only `storage.objects.list`. Terraform's GCS backend +needs that bucket-level permission before it can access the exact state and lock +objects protected by the conditional object-admin binding. + +Production also requires one exact in-place update to +`google_iam_workload_identity_pool_provider.github[0]` so the existing deploy +identity accepts `operate-relay-asia-admission.yml`; staging's shared provider +already accepts repository workflows. Reject every other change. Apply only +that saved plan, then require the same targeted plan to be empty. Publish the two +`github_relay_asia_topology_*` outputs as the matching staging or production +GitHub environment variables documented in `.github/workflows/README.md`. + +Apply observability separately from IAM and topology. The topology identity +has no IAM, logging-metric, alert-policy, Cloud SQL, DNS, certificate, global +IP, or deletion permission. Its read role includes `serviceusage.services.list` +because the Google provider lists managed APIs while refreshing targeted plans. +Its mutation role includes `compute.networks.updatePolicy`, which Compute requires +to attach the reviewed Asia subnet and router to the existing Relay VPC. +It also includes `compute.healthChecks.useReadOnly`, which backend creation requires +to reference the existing Relay readiness health check. +Managed-group creation additionally requires `compute.instanceGroups.create`; adding +that group as a backend requires `compute.instanceGroups.use` and `compute.instances.use`. + +Bootstrap the staging Asia proof identity separately before its director roll. +Its targeted plan contains only the proof provider, service account, +workload-identity binding, logging/monitoring viewer bindings, and two outputs. The provider +accepts only `prove-relay-asia-staging.yml` on `main` in the staging environment; +the account has no Compute, Cloud SQL, Secret Manager, Terraform-state, or +Cloud Run mutation permission. Publish its provider and account outputs as +`STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER` and +`STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT`, then deploy the compatible +staging director so it accepts that exact account for bounded capacity routes. + +## Relay regional-placement switch bootstrap + +Before the first director deployment that references the regional-placement +switch, apply its Secret Manager resources with operator credentials. Save a +targeted plan containing exactly these six additions and no other changes: + +1. `google_secret_manager_secret.relay_regional_placement_enabled` +2. `google_secret_manager_secret_version.relay_regional_placement_enabled` +3. `google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor` +4. `google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor[0]` +5. `google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder[0]` +6. `google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer[0]` + +Pass the exact environment tfvars, apply only +the saved plan, then require the same targeted plan to be empty. Verify the +runtime and deploy identities can access the secret without printing its value, and the deploy +identity can read version metadata without gaining broader mutation rights. +Only then deploy a director revision. Every director revision pins one exact +numeric secret version; the audited director workflow preserves the serving +version by default and creates a new boolean version only for an explicit +enable or disable. The traffic move is therefore the switch commit, and a +failed candidate cannot change the value used by serving instances. +Terraform reads and preserves the currently served director's exact numeric +version, falling back to the bootstrap version only before the setting exists. +It still owns the secret name and every environment field; an unrelated apply +therefore cannot revert a later audited switch version. + +## Relay director runtime identity bootstrap + +Before the regional-rehome director rollout, create the distinct director +runtime identity with operator credentials. Reinitialize the exact environment +backend, export a fresh `GOOGLE_OAUTH_ACCESS_TOKEN` without printing it, pass +the environment tfvars, and save a targeted +plan containing only the applicable resources below: + +1. `google_service_account.relay_director_runtime` +2. `google_project_iam_member.relay_director_runtime_cloudsql_client` +3. `google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor` +4. `google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor` +5. `google_secret_manager_secret_iam_member.relay_database_url_director_accessor` +6. `google_service_account_iam_member.github_relay_director_runtime_service_account_user[0]` +7. `google_iam_workload_identity_pool_provider.github[0]` in production when its + condition adds the exact regional-rehome and same-cap workflow/job pairs +8. `google_iam_workload_identity_pool_provider.github_production_relay_capacity[0]` + in production when its condition adds the exact same-cap workflow/job pair + +Reject Cloud Run, GCE template, database, network, DNS, or any other change. +Apply only the reviewed saved plan, then require the same targeted plan to be +empty. Publish `relay_director_runtime_service_account` and +`relay_runtime_service_account` as the matching GitHub environment variables +documented in `.github/workflows/README.md`. The identity bootstrap does not +authorize a rollout by itself; use the one-time director workflow mode so both +candidate and rollback revisions move together while rehoming remains durably +disabled. + +## Usage + +```sh +pnpm infra:init --env staging +pnpm infra:plan --env staging +pnpm infra:apply --env staging +``` + +Add `--root foundation` or `--root apps` for the other two roots; the default is the relay root. +On a greenfield project apply foundation before either of the others. + +Run staging first. Production should only follow after staging has a successful `/health` smoke test. + +## Relay staging topology + +The stable Cloud Run service is the director. Staging uses fixed-one GCE cells so its data plane +matches production; Cloud Run stamped cells are no longer retained in the live environment. + +Staging is intentionally allowed to drift to a powered-off runtime state between internal test +windows. Use the `Power Relay Staging` GitHub Actions workflow to inspect, wake, or sleep it. A +normal `pnpm infra:apply --env staging` refuses while Cloud SQL is stopped or any staging MIG is +scaled below its Terraform-owned size of one. Dispatch `wake` with `wake-cells: all`, wait for its +health checks, and only then apply a reviewed staging plan. Do not use Terraform to wake staging: +that can mix infrastructure changes with a partial power transition. + +The staging tfvars keep both Cloud Run services at zero minimum instances. Requests wake the auth +service and director when SQL is running; the power workflow separately controls SQL and the GCE +MIGs. The staging-only GitHub service-account role can resize those MIGs and change the SQL +activation policy. Terraform does not create that role in production. + +## Relay GCE production data plane + +`relay_gce_domain` creates the shared private network/NAT, LB address, and Certificate Manager +wildcard authorization used by fixed-one GCE cell MIGs. Cells use exact hosts one label below the +domain, such as `c1.relay-staging.onorca.dev`; future cells therefore reuse one DNS-only wildcard +A record while the HTTPS URL map still admits only Terraform-configured exact hosts. + +After the foundation apply, publish both Terraform outputs and leave them in place for renewal: + +1. `relay_gce_certificate_dns_authorization`: the exact Certificate Manager CNAME. +2. `relay_gce_wildcard_dns_record`: the DNS-only wildcard A record to the reserved LB address. + +Every `relay_gce_cells` entry is one durable cell generation and must pin both its exact COS boot +image and its Artifact Registry relay image. Terraform creates one private COS instance template, one size-one zonal +MIG, and one backend service for that exact host. The MIG uses `RECREATE`, zero surge, and one +unavailable worker; `/health` alone drives autoheal while SQL/JWKS-backed `/ready` controls LB +admission. The backend timeout is 86,400 seconds with connection draining, and the URL map aborts +unknown wildcard hosts before they reach a worker. The startup script obtains short-lived metadata +credentials, fetches the two relay secrets without logging them, and runs a digest-pinned Cloud SQL +Auth Proxy beside the digest-pinned relay image. + +The primary `us-central1` subnet, router, and NAT retain their original +Terraform addresses. `relay_gce_additional_region_subnetwork_cidrs` creates +only additive regional resources; cells select the subnet from their declared +region. Every cell also declares an explicit database pool maximum in startup +metadata and deployment outputs. The initial Asia shape is `e2-standard-4`, +3,000 physical connections, 60 unobserved connections, 6,000 request units, +and a database pool maximum of 10. + +Provision the complete identical Asia wave in one `Deploy Relay Asia Topology` +saved plan. Its validator permits only the additive subnet/router/NAT, reviewed +cell templates/MIGs/backends, and exact shared URL-map host additions. It +rejects deletes, replacements, loss of an existing host route, US-resource +changes, and unrelated drift. Do not add production C27-C29 until the +compatible image has been published and each entry can pin its immutable +digest. + +Topology creation intentionally does not apply the director resource. Once all +MIGs and backends are healthy, register every new cell atomically as +migration-only through `Operate Relay Asia Admission` with the exact live +selector generation and a durable attempt ID. Only then may a director +deployment list the new cells. Verify that configuration and fresh heartbeats +before using the same workflow to promote the canary. A failed canary returns to +migration-only; do not delete the Asia network during rollout recovery. + +`relay_gce_cells` takes precedence in the director's configured-cell list. Adding or replacing a +generation requires a new map key and hostname; do not change an active cell's image in place. Run +`terraform fmt -check -recursive` and `terraform validate` locally; the same non-credentialed +checks run on every pull request. + +GCE deployments must add a distinct cell ID, host, backend, and MIG for every candidate generation; +never update an existing generation's image behind its origin. + +For a post-launch worker replacement, first publish an immutable image with the production image +workflow. Add that digest as a distinct `relay_gce_cells` entry with +`initially_enabled = false`, review/apply the Terraform change, and deploy the compatible director. +The production candidate workflow then reads the remote-state topology and defaults to a read-only +preflight. It verifies the exact TLS origin, `/health`, dependency-backed `/ready`, authenticated +heartbeat, served digest, private fixed-one MIG, runtime identity, dedicated backend, 86,400-second +timeout, and authoritative request-unit headroom. `execute` requires the literal `EVACUATE` +confirmation, disables the source only after preflight, enables the candidate, performs bounded +target-first evacuation, drains the exact source origin, and verifies aggregate completion. Keep +both source and candidate Terraform routes until a later reviewed removal proves the old origin has +no assignments, activity leases, or migrations. + +After any failure following target registration, use `audit` before `recover-forward`; never retry +`execute` or reverse admission. Forward recovery retries only bounded idempotent status operations. +If every remaining migration belongs to a registered target whose desktop is currently offline, it +emits `candidate_forward_pending` and stops without retiring those rows. Keep both origins intact +and rerun recovery only after a fresh audit shows target controls have returned. + +The production multi-target workflow is the reviewed path for evacuations that +need more than one candidate. It enforces deterministic serialized quotas, +target connection ceilings, and the oldest-migration lease gate before drain. +After selector generation 1, add new disabled targets without changing the +director resource in the targeted apply. Deploy the selector-version-2 +director, apply only the new cell templates, MIGs, backends, and URL-map +routes, then use `add-migration-cells` to register their exact configs as +migration-only in one selector generation. A single additive target is valid; +ordinary evacuation and supersession retain their multi-target requirements. +Use `retire-migration-cell` with an exact attempt ID to move one +migration-only cell to existing-only before its reviewed fence. +The director does not depend on the GCE forwarding-rule graph; keep director +configuration plans scoped away from immutable cell generations. +Its guarded `fence-source` mode applies an exact private Terraform saved plan +for a fully quiescent cell already listed in `relay_gce_fenced_cells`. The plan +must contain only that MIG's in-place target-size change from one to zero, so +the origin, backend, and generation remain retained. An interrupted apply is +always recovered forward unless Terraform state, live GCE state, and operation +history prove it never began. `abort-fence-source` records that proven +pre-apply abort; remove the cell from the fence set only in a later reviewed +commit. Never resize a production relay MIG directly. +Before the first fencing workflow rollout, apply this schema with an empty +fence set so remote-state topology contains `generation_identity`, +`fenced`, and `desired_target_size`. Only then commit a cell ID into the +production fence set. +The same workflow's `supersede-target` mode is the only supported path for a +failed registered target: it proves the failed MIG is zero with no instances +before recording an exact-incarnation fence and publishing newer epochs. +It invokes an IAM-authenticated max-one Cloud Run broker. The broker runtime +alone can access the exact state/saved-plan/lease object prefixes and update a +Relay MIG; the GitHub requester can read aggregate safety evidence and invoke +that service but cannot perform either mutation directly. diff --git a/cloud/infra/terraform/backend/production.hcl b/cloud/infra/terraform/backend/production.hcl new file mode 100644 index 00000000000..e482b091273 --- /dev/null +++ b/cloud/infra/terraform/backend/production.hcl @@ -0,0 +1,3 @@ +bucket = "onorca-cloud-terraform-state" +prefix = "terraform/state" + diff --git a/cloud/infra/terraform/backend/staging.hcl b/cloud/infra/terraform/backend/staging.hcl new file mode 100644 index 00000000000..fbd0f0c17b2 --- /dev/null +++ b/cloud/infra/terraform/backend/staging.hcl @@ -0,0 +1,3 @@ +bucket = "onorca-cloud-staging-terraform-state" +prefix = "terraform/state" + diff --git a/cloud/infra/terraform/environments/production.tfvars b/cloud/infra/terraform/environments/production.tfvars new file mode 100644 index 00000000000..60d36e3d832 --- /dev/null +++ b/cloud/infra/terraform/environments/production.tfvars @@ -0,0 +1,418 @@ +project_id = "onorca-cloud" +environment = "production" +name_prefix = "orca-cloud" +region = "us-central1" + +artifact_repository_id = "orca-cloud" + +# Dual accept while the relay source moves to the public stablyai/orca repository: the same +# workflows are trusted from both repos, and the public copies carry a `cloud-` file prefix. +# Remove this entry once the private workflows are retired and point github_owner/github_repo, +# github_repo_id, and github_owner_id at the surviving repository. +github_accepted_repositories = [ + { + owner = "stablyai" + repo = "orca" + repo_id = "1183888342" + owner_id = "127256420" + workflow_file_prefix = "cloud-" + } +] + +# Our first-party auth service. auth.onorca.dev is PropelAuth's prod domain, so +# our service lives at login.onorca.dev (desktop points ORCA_CLOUD_API_URL here). +auth_base_url = "https://login.onorca.dev" + +relay_cloud_run_service_name = "orca-cloud-relay" +relay_base_url = "https://relay.onorca.dev" +# Why: public admission is a per-instance semaphore, so fleet assignment capacity is +# concurrency x instances. Scaling to 2 instances took placement failures 35% -> 70%. +relay_min_instances = 5 +relay_max_instances = 5 +# Production cells run only on fixed-one GCE MIGs; Cloud Run remains the director. +relay_cells = {} +manage_relay_domain_mapping = true + +# Production GCE cells use exact hosts such as c1.relay.onorca.dev. +# The wildcard only handles DNS/TLS; the load balancer rejects unknown hosts. +relay_gce_domain = "relay.onorca.dev" +relay_gce_subnetwork_cidr = "10.42.0.0/24" +relay_gce_additional_region_subnetwork_cidrs = { + "asia-east2" = "10.42.1.0/24" +} +relay_gce_fenced_cells = ["production-gce-c1", "production-gce-c2", "production-gce-c3", "production-gce-c6", "production-gce-c11", "production-gce-c12"] +# Initial cells stay admission-disabled until production preflight and go-live approval. +relay_gce_cells = { + "production-gce-c1" = { + hostname = "c1" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" + initially_enabled = false + } + "production-gce-c2" = { + hostname = "c2" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" + initially_enabled = false + } + "production-gce-c3" = { + hostname = "c3" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:19ef4e6e4a043f63d78011d1c29a395a9002ec077d03ee6931f25479fe66f349" + initially_enabled = false + } + # Distinct origins let each existing cell drain without an in-place image swap. + "production-gce-c4" = { + hostname = "c4" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" + initially_enabled = false + } + "production-gce-c5" = { + hostname = "c5" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d" + initially_enabled = false + } + "production-gce-c6" = { + hostname = "c6" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" + initially_enabled = false + } + "production-gce-c7" = { + hostname = "c7" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c8" = { + hostname = "c8" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c9" = { + hostname = "c9" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + # Canary for the control-activation fence (PR #207, main b253fcd). + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c10" = { + hostname = "c10" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c11" = { + hostname = "c11" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:e592371013188b8297e395979c70a8b42c39f4bb5f90b01190f0778279cbaef5" + initially_enabled = false + connection_hard_cap = 600 + connection_unobserved_bound = 60 + } + "production-gce-c12" = { + hostname = "c12" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:3d8b388dcbf190be20491ce9c14eeafa0dccd0afbb2725712f6f9d9a754838dc" + initially_enabled = false + connection_hard_cap = 600 + connection_unobserved_bound = 60 + } + "production-gce-c13" = { + hostname = "c13" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c14" = { + hostname = "c14" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c15" = { + hostname = "c15" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c16" = { + hostname = "c16" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c17" = { + hostname = "c17" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d" + initially_enabled = false + connection_hard_cap = 600 + connection_unobserved_bound = 60 + } + # Canary for halved control lease renewal (PR #253, main 11256b5). Chosen as the + # smallest live cell: ~9 connections, so a replace costs 9 reconnects, not ~400. + "production-gce-c18" = { + hostname = "c18" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d" + initially_enabled = false + connection_hard_cap = 600 + connection_unobserved_bound = 60 + } + "production-gce-c19" = { + hostname = "c19" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c20" = { + hostname = "c20" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + # First full-size cell on the halved lease renewal, after the C18 canary measured + # 9.15 -> 5.32 queries per connection with zero failures. C21 also carries the + # control-close churn we still need to attribute to a client build. + "production-gce-c21" = { + hostname = "c21" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c22" = { + hostname = "c22" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c23" = { + hostname = "c23" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c24" = { + hostname = "c24" + zone = "us-central1-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c25" = { + hostname = "c25" + zone = "us-central1-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c26" = { + hostname = "c26" + zone = "us-central1-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "production-gce-c27" = { + hostname = "c27" + region = "asia-east2" + zone = "asia-east2-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 6000 + database_pool_max = 10 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 3000 + connection_unobserved_bound = 60 + } + "production-gce-c28" = { + hostname = "c28" + region = "asia-east2" + zone = "asia-east2-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 6000 + database_pool_max = 10 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 3000 + connection_unobserved_bound = 60 + } + "production-gce-c29" = { + hostname = "c29" + region = "asia-east2" + zone = "asia-east2-c" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 6000 + database_pool_max = 10 + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 3000 + connection_unobserved_bound = 60 + } +} + +relay_region_rehome_source_cell_ids = [ + "production-gce-c7", + "production-gce-c8", + "production-gce-c9", + "production-gce-c10", + "production-gce-c13", + "production-gce-c14", + "production-gce-c15", + "production-gce-c16", + "production-gce-c19", + "production-gce-c20", + "production-gce-c21", + "production-gce-c22", + "production-gce-c23", + "production-gce-c24", + "production-gce-c25", + "production-gce-c26" +] + +# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply +# was otherwise going to strip it from every policy, leaving the alerts firing at nobody. +relay_alert_notification_channels = ["projects/onorca-cloud/notificationChannels/4879431412695417284"] diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars new file mode 100644 index 00000000000..3ee108fe874 --- /dev/null +++ b/cloud/infra/terraform/environments/staging.tfvars @@ -0,0 +1,91 @@ +project_id = "onorca-cloud-staging" +environment = "staging" +name_prefix = "orca-cloud-staging" +region = "us-central1" + +artifact_repository_id = "orca-cloud" + +# Dual accept while the relay source moves to the public stablyai/orca repository: the same +# workflows are trusted from both repos, and the public copies carry a `cloud-` file prefix. +# Remove this entry once the private workflows are retired and point github_owner/github_repo, +# github_repo_id, and github_owner_id at the surviving repository. +github_accepted_repositories = [ + { + owner = "stablyai" + repo = "orca" + repo_id = "1183888342" + owner_id = "127256420" + workflow_file_prefix = "cloud-" + } +] + +auth_base_url = "https://auth-staging.onorca.dev" + +relay_cloud_run_service_name = "orca-cloud-relay-staging" +relay_staging_power_auth_service_name = "orca-cloud-auth-staging" +relay_base_url = "https://relay-staging.onorca.dev" +relay_min_instances = 0 +relay_max_instances = 2 +# Staging now exercises the production-shaped GCE data plane exclusively. +relay_cells = {} +# Keep the stable director mapping Terraform-owned while removing cell mappings. +manage_relay_domain_mapping = true + +# GCE cells use exact hosts below this wildcard, for example +# c1.relay-staging.onorca.dev. Cloudflare records remain out-of-band. +relay_gce_domain = "relay-staging.onorca.dev" +relay_gce_subnetwork_cidr = "10.42.0.0/24" +relay_gce_additional_region_subnetwork_cidrs = { + "asia-east2" = "10.42.1.0/24" +} +relay_gce_fenced_cells = [] +relay_gce_cells = { + "staging-gce-c1" = { + hostname = "c1" + zone = "us-central1-b" + machine_type = "e2-standard-2" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:2d0f6e6db2b0eb9d6aba188698de8330f8c30b4e76badfcf0fac3f3eb9508a87" + } + "staging-gce-c2" = { + hostname = "c2" + zone = "us-central1-c" + machine_type = "e2-standard-2" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:1239830d0946dc92ded3c9edde1c0b827f584a7a2be5c177beed900056d76f69" + connection_hard_cap = 600 + connection_unobserved_bound = 60 + } + "staging-gce-c3" = { + hostname = "c3" + zone = "us-central1-a" + machine_type = "e2-standard-2" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" + capacity_requests = 4000 + image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:9fba2a189ab3fa29853800830e77c7551ab3aaa8f43f3cd9adbdea28b876a8b9" + initially_enabled = false + connection_hard_cap = 1000 + connection_unobserved_bound = 60 + } + "staging-gce-c4" = { + hostname = "c4" + region = "asia-east2" + zone = "asia-east2-a" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 6000 + database_pool_max = 10 + image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" + initially_enabled = false + connection_hard_cap = 3000 + connection_unobserved_bound = 60 + } +} + +relay_region_rehome_source_cell_ids = ["staging-gce-c2", "staging-gce-c3"] diff --git a/cloud/infra/terraform/outputs.tf b/cloud/infra/terraform/outputs.tf new file mode 100644 index 00000000000..220aa5cf94f --- /dev/null +++ b/cloud/infra/terraform/outputs.tf @@ -0,0 +1,191 @@ +output "github_deploy_service_account" { + value = try(google_service_account.github_deploy[0].email, null) + description = "Service account email to use in the GitHub Actions deploy workflow." +} + +output "github_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github[0].name, null) + description = "Workload Identity provider resource name for GitHub Actions." +} + +output "github_relay_monitor_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_monitor[0].name, null) + description = "Exact-workflow provider for PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER." +} + +output "github_relay_monitor_service_account" { + value = try(google_service_account.github_monitor[0].email, null) + description = "Read-only identity for PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT." +} + +output "github_relay_fence_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_fence[0].name, null) + description = "Exact-workflow provider for PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER." +} + +output "github_relay_fence_service_account" { + value = try(google_service_account.github_fence[0].email, null) + description = "Narrow fencing identity for PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT." +} + +output "github_staging_relay_capacity_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_staging_relay_capacity[0].name, null) + description = "Exact-workflow provider for STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER." +} + +output "github_staging_relay_capacity_service_account" { + value = try(google_service_account.github_staging_relay_capacity[0].email, null) + description = "Narrow transition identity for STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT." +} + +output "github_staging_relay_deploy_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_staging_relay_deploy[0].name, null) + description = "Exact-workflow provider for STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER." +} + +output "github_staging_relay_deploy_service_account" { + value = try(google_service_account.github_staging_relay_deploy[0].email, null) + description = "Account for STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT." +} + +output "github_production_relay_capacity_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_production_relay_capacity[0].name, null) + description = "Exact-workflow provider for PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER." +} + +output "github_production_relay_capacity_service_account" { + value = try(google_service_account.github_production_relay_capacity[0].email, null) + description = "Narrow transition identity for PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT." +} + +output "github_relay_asia_topology_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_relay_asia_topology[0].name, null) + description = "Workflow-bound provider for validated additive Relay Asia topology plans." +} + +output "github_relay_asia_topology_service_account" { + value = try(google_service_account.github_relay_asia_topology[0].email, null) + description = "Dedicated identity for validated additive Relay Asia topology plans." +} + +output "github_relay_asia_proof_workload_identity_provider" { + value = try(google_iam_workload_identity_pool_provider.github_relay_asia_proof[0].name, null) + description = "Workflow-bound staging Relay Asia proof Workload Identity provider." +} + +output "github_relay_asia_proof_service_account" { + value = try(google_service_account.github_relay_asia_proof[0].email, null) + description = "Least-privilege staging Relay Asia proof service account." +} + +output "relay_fence_broker_service_uri" { + value = try(google_cloud_run_v2_service.relay_fence_broker[0].uri, null) + description = "IAM-authenticated private Relay fence broker URI." +} + +output "relay_fence_broker_service_account" { + value = try(google_service_account.relay_fence_broker[0].email, null) + description = "Runtime identity that owns exact Relay fence mutations." +} + + +output "relay_cloud_run_service_uri" { + value = google_cloud_run_v2_service.relay.uri + description = "Default relay service URI for pre-domain smoke tests." +} + +output "relay_runtime_service_account" { + value = google_service_account.relay_runtime.email + description = "Runtime identity for stamped Relay cells." +} + +output "relay_director_runtime_service_account" { + value = google_service_account.relay_director_runtime.email + description = "Runtime and regional rehoming caller identity for the Relay director." +} + +output "relay_cell_cloud_run_service_uris" { + value = { + for cell_id, service in google_cloud_run_v2_service.relay_cell : + cell_id => service.uri + } + description = "Native Cloud Run URIs for stamped relay cells." +} + +output "relay_database_name" { + value = google_sql_database.relay.name + description = "Database isolated for durable relay state." +} + +output "relay_gce_load_balancer_ip" { + value = try(google_compute_global_address.relay_gce[0].address, null) + description = "Reserved IPv4 address for the shared GCE relay HTTPS load balancer." +} + +output "relay_gce_wildcard_dns_record" { + value = var.relay_gce_domain == "" ? null : { + name = "*.${var.relay_gce_domain}" + type = "A" + data = try(google_compute_global_address.relay_gce[0].address, null) + } + description = "DNS-only wildcard record that routes future cell hosts to the shared LB." +} + +output "relay_gce_certificate_dns_authorization" { + value = try(google_certificate_manager_dns_authorization.relay_gce[0].dns_resource_record[0], null) + description = "Certificate Manager DNS record that must remain published for renewal." +} + +output "relay_gce_cell_origins" { + value = local.relay_gce_cell_urls + description = "Exact public origins admitted by the shared GCE relay load balancer." +} + +output "relay_gce_cell_instance_groups" { + value = { + for cell_id, manager in google_compute_instance_group_manager.relay_gce_cell : + cell_id => manager.instance_group + } + description = "Terraform-sized managed instance groups backing each GCE relay cell." +} + +output "relay_gce_cell_backend_services" { + value = { + for cell_id, backend in google_compute_backend_service.relay_gce_cell : + cell_id => backend.id + } + description = "Non-overlapping backend service for each exact relay cell host." +} + +output "relay_gce_cell_deployments" { + value = { + for cell_id, cell in var.relay_gce_cells : cell_id => { + origin = local.relay_gce_cell_urls[cell_id] + region = cell.region + zone = cell.zone + mig_name = google_compute_instance_group_manager.relay_gce_cell[cell_id].name + instance_group = google_compute_instance_group_manager.relay_gce_cell[cell_id].instance_group + backend_name = google_compute_backend_service.relay_gce_cell[cell_id].name + backend_id = google_compute_backend_service.relay_gce_cell[cell_id].id + url_map_name = google_compute_url_map.relay_gce[0].name + generation_identity = google_compute_instance_template.relay_gce_cell[cell_id].self_link + image = cell.image + capacity_requests = cell.capacity_requests + database_pool_max = cell.database_pool_max + connection_hard_cap = cell.connection_hard_cap + connection_unobserved_bound = cell.connection_unobserved_bound + initially_enabled = cell.initially_enabled + fenced = contains(var.relay_gce_fenced_cells, cell_id) + desired_target_size = local.relay_gce_cell_target_sizes[cell_id] + target_size = google_compute_instance_group_manager.relay_gce_cell[cell_id].target_size + } + } + description = "Non-secret candidate deployment topology consumed by the GCE preflight workflow." + + precondition { + condition = alltrue([ + for cell_id in var.relay_gce_fenced_cells : contains(keys(var.relay_gce_cells), cell_id) + ]) + error_message = "relay_gce_fenced_cells may contain only configured relay_gce_cells keys." + } +} diff --git a/cloud/infra/terraform/relay-asia-proof-iam.tf b/cloud/infra/terraform/relay-asia-proof-iam.tf new file mode 100644 index 00000000000..e1ff687677b --- /dev/null +++ b/cloud/infra/terraform/relay-asia-proof-iam.tf @@ -0,0 +1,79 @@ +locals { + create_relay_asia_proof_identity = ( + local.relay_create_github_deploy_identity && var.environment == "staging" + ) + github_relay_asia_proof_workflow_file = "prove-relay-asia-staging.yml" + github_relay_asia_proof_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "assertion.workflow_ref == '${prefix}${local.github_relay_asia_proof_workflow_file}@refs/heads/main'" + ] + relay_asia_proof_service_account_email = try( + google_service_account.github_relay_asia_proof[0].email, + "" + ) +} + +resource "google_iam_workload_identity_pool_provider" "github_relay_asia_proof" { + count = local.create_relay_asia_proof_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-relay-asia-proof" + display_name = "GitHub Relay Asia staging proof" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.environment" = "assertion.environment" + "attribute.event_name" = "assertion.event_name" + "attribute.ref" = "assertion.ref" + "attribute.repository" = "assertion.repository" + "attribute.repository_id" = "assertion.repository_id" + "attribute.repository_owner_id" = "assertion.repository_owner_id" + "attribute.workflow_ref" = "assertion.workflow_ref" + "attribute.relay_ops_identity" = "'staging-asia-proof'" + } + + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'staging'", + "assertion.event_name == 'workflow_dispatch'", + local.relay_github_workflow_conditions["github_relay_asia_proof"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account" "github_relay_asia_proof" { + count = local.create_relay_asia_proof_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-aproof" + display_name = "Orca Relay Asia staging proof" + description = "Reads staging telemetry and performs only Relay's bounded Asia proof operations." +} + +resource "google_service_account_iam_member" "github_relay_asia_proof_workload_identity_user" { + count = local.create_relay_asia_proof_identity ? 1 : 0 + + service_account_id = google_service_account.github_relay_asia_proof[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/staging-asia-proof" +} + +resource "google_project_iam_member" "github_relay_asia_proof_logging_viewer" { + count = local.create_relay_asia_proof_identity ? 1 : 0 + + project = var.project_id + role = "roles/logging.viewer" + member = google_service_account.github_relay_asia_proof[0].member +} + +resource "google_project_iam_member" "github_relay_asia_proof_monitoring_viewer" { + count = local.create_relay_asia_proof_identity ? 1 : 0 + + project = var.project_id + role = "roles/monitoring.viewer" + member = google_service_account.github_relay_asia_proof[0].member +} diff --git a/cloud/infra/terraform/relay-asia-topology-iam.tf b/cloud/infra/terraform/relay-asia-topology-iam.tf new file mode 100644 index 00000000000..eea36e26247 --- /dev/null +++ b/cloud/infra/terraform/relay-asia-topology-iam.tf @@ -0,0 +1,207 @@ +locals { + create_relay_asia_topology_identity = local.relay_create_github_deploy_identity + github_relay_asia_topology_workflow_file = "deploy-relay-asia-topology.yml" + github_relay_asia_topology_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "assertion.workflow_ref == '${prefix}${local.github_relay_asia_topology_workflow_file}@refs/heads/main'" + ] +} + +resource "google_iam_workload_identity_pool_provider" "github_relay_asia_topology" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-relay-asia" + display_name = "GitHub Relay Asia topology" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.environment" = "assertion.environment" + "attribute.event_name" = "assertion.event_name" + "attribute.ref" = "assertion.ref" + "attribute.repository" = "assertion.repository" + "attribute.repository_id" = "assertion.repository_id" + "attribute.repository_owner_id" = "assertion.repository_owner_id" + "attribute.workflow_ref" = "assertion.workflow_ref" + "attribute.relay_ops_identity" = "'${var.environment}-asia-topology'" + } + + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == '${var.environment}'", + "assertion.event_name == 'workflow_dispatch'", + local.relay_github_workflow_conditions["github_relay_asia_topology"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account" "github_relay_asia_topology" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-asia" + display_name = "Orca Relay Asia topology" + description = "Applies only validated additive Relay Asia topology plans." +} + +resource "google_service_account_iam_member" "github_relay_asia_topology_workload_identity_user" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + service_account_id = google_service_account.github_relay_asia_topology[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/${var.environment}-asia-topology" +} + +resource "google_project_iam_custom_role" "github_relay_asia_topology_mutation" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + role_id = "orcaRelayAsiaTopology" + title = "Orca Relay Asia topology" + description = "Creates additive Relay Asia network and cell topology and updates its shared URL map." + permissions = [ + "compute.backendServices.create", + "compute.backendServices.get", + "compute.backendServices.update", + "compute.backendServices.use", + "compute.disks.create", + "compute.globalOperations.get", + "compute.healthChecks.use", + "compute.healthChecks.useReadOnly", + "compute.images.useReadOnly", + "compute.instanceGroupManagers.create", + "compute.instanceGroupManagers.get", + "compute.instanceGroupManagers.update", + "compute.instanceGroups.create", + "compute.instanceGroups.get", + "compute.instanceGroups.use", + "compute.instances.create", + "compute.instances.setLabels", + "compute.instances.setMetadata", + "compute.instances.setTags", + "compute.instances.use", + "compute.instanceTemplates.create", + "compute.instanceTemplates.get", + "compute.instanceTemplates.useReadOnly", + "compute.networks.get", + "compute.networks.updatePolicy", + "compute.networks.use", + "compute.regionOperations.get", + "compute.routers.create", + "compute.routers.get", + "compute.routers.update", + "compute.subnetworks.create", + "compute.subnetworks.get", + "compute.subnetworks.setPrivateIpGoogleAccess", + "compute.subnetworks.use", + "compute.urlMaps.get", + "compute.urlMaps.update", + "compute.zoneOperations.get" + ] +} + +resource "google_project_iam_member" "github_relay_asia_topology_mutation" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + role = google_project_iam_custom_role.github_relay_asia_topology_mutation[0].id + member = google_service_account.github_relay_asia_topology[0].member +} + +resource "google_project_iam_custom_role" "github_relay_asia_topology_read" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + role_id = "orcaRelayAsiaTopologyRead" + title = "Orca Relay Asia topology read" + description = "Refreshes only resource types required by validated Relay Asia topology plans." + permissions = [ + "artifactregistry.repositories.get", + "cloudsql.instances.get", + "compute.backendServices.get", + "compute.healthChecks.get", + "compute.instanceGroupManagers.get", + "compute.instanceGroups.get", + "compute.instanceTemplates.get", + "compute.instances.get", + "compute.networks.get", + "compute.routers.get", + "compute.subnetworks.get", + "compute.urlMaps.get", + "iam.serviceAccounts.get", + "iam.serviceAccounts.getIamPolicy", + "resourcemanager.projects.get", + "resourcemanager.projects.getIamPolicy", + "run.revisions.get", + "run.services.get", + "secretmanager.secrets.get", + "secretmanager.secrets.getIamPolicy", + "serviceusage.services.get", + "serviceusage.services.list" + ] +} + +resource "google_project_iam_member" "github_relay_asia_topology_read" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + role = google_project_iam_custom_role.github_relay_asia_topology_read[0].id + member = google_service_account.github_relay_asia_topology[0].member +} + +resource "google_artifact_registry_repository_iam_member" "github_relay_asia_topology_artifact_reader" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + location = var.region + repository = var.artifact_repository_id + role = "roles/artifactregistry.reader" + member = google_service_account.github_relay_asia_topology[0].member +} + +resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + bucket = "${var.project_id}-terraform-state" + role = "roles/storage.objectAdmin" + member = google_service_account.github_relay_asia_topology[0].member + + condition { + title = "relay_asia_topology_state" + description = "Limits the Asia topology workflow to the environment Terraform state and lock." + expression = join(" || ", [ + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" + ]) + } +} + +resource "google_project_iam_custom_role" "github_relay_asia_topology_state_list" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + project = var.project_id + role_id = "orcaRelayAsiaStateList" + title = "Orca Relay Asia state list" + description = "Lists the environment state bucket so Terraform can initialize its backend." + permissions = ["storage.objects.list"] +} + +resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state_list" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + bucket = "${var.project_id}-terraform-state" + role = google_project_iam_custom_role.github_relay_asia_topology_state_list[0].id + member = google_service_account.github_relay_asia_topology[0].member +} + +resource "google_service_account_iam_member" "github_relay_asia_topology_runtime_user" { + count = local.create_relay_asia_topology_identity ? 1 : 0 + + service_account_id = google_service_account.relay_runtime.name + role = "roles/iam.serviceAccountUser" + member = google_service_account.github_relay_asia_topology[0].member +} diff --git a/cloud/infra/terraform/relay-database.tf b/cloud/infra/terraform/relay-database.tf new file mode 100644 index 00000000000..5dbbd71aa31 --- /dev/null +++ b/cloud/infra/terraform/relay-database.tf @@ -0,0 +1,54 @@ +# Relay credentials and assignment state share the existing Cloud SQL instance +# with auth, but use an isolated database and principal. +resource "google_sql_database" "relay" { + project = var.project_id + name = "orca_relay" + instance = local.relay_database_instance_name +} + +resource "random_password" "relay_database" { + length = 32 + special = false +} + +resource "google_sql_user" "relay" { + project = var.project_id + name = "orca_relay" + instance = local.relay_database_instance_name + password = random_password.relay_database.result +} + +resource "google_secret_manager_secret" "relay_database_url" { + project = var.project_id + secret_id = "orca-cloud-relay-database-url" + labels = local.relay_shared_labels + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "relay_database_url" { + secret = google_secret_manager_secret.relay_database_url.id + secret_data = format( + "postgresql://%s:%s@/%s?host=/cloudsql/%s", + google_sql_user.relay.name, + random_password.relay_database.result, + google_sql_database.relay.name, + local.relay_database_connection_name + ) +} + +resource "google_secret_manager_secret_iam_member" "relay_database_url_accessor" { + project = var.project_id + secret_id = google_secret_manager_secret.relay_database_url.secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_runtime.member +} + +resource "google_secret_manager_secret_iam_member" "relay_database_url_director_accessor" { + project = var.project_id + secret_id = google_secret_manager_secret.relay_database_url.secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_director_runtime.member +} diff --git a/cloud/infra/terraform/relay-dns.tf b/cloud/infra/terraform/relay-dns.tf new file mode 100644 index 00000000000..6e1895da925 --- /dev/null +++ b/cloud/infra/terraform/relay-dns.tf @@ -0,0 +1,47 @@ +# Relay custom domains. The Cloud Run domain mapping is the whole story here: Google issues and +# renews the certificate, and the DNS records that point at ghs.googlehosted.com are managed +# outside this root (the auth and artifact records moved to the apps root with their services). + +locals { + relay_fqdn = replace(replace(var.relay_base_url, "https://", ""), "http://", "") + relay_cell_fqdns = { + for cell_id, cell in var.relay_cells : + cell_id => replace(replace(cell.url, "https://", ""), "http://", "") + } +} + +resource "google_cloud_run_domain_mapping" "relay" { + count = var.manage_relay_domain_mapping ? 1 : 0 + location = var.region + name = local.relay_fqdn + + metadata { + namespace = var.project_id + } + + spec { + route_name = google_cloud_run_v2_service.relay.name + } + + # gcloud-created mappings report an empty legacy certificate_mode even + # though Google provisions the same automatic certificate; replacing it + # would reset issuance for no behavioral change. + lifecycle { + ignore_changes = [spec[0].certificate_mode] + } +} + +resource "google_cloud_run_domain_mapping" "relay_cell" { + for_each = var.manage_relay_domain_mapping ? var.relay_cells : {} + + location = var.region + name = local.relay_cell_fqdns[each.key] + + metadata { + namespace = var.project_id + } + + spec { + route_name = google_cloud_run_v2_service.relay_cell[each.key].name + } +} diff --git a/cloud/infra/terraform/relay-fence-broker.tf b/cloud/infra/terraform/relay-fence-broker.tf new file mode 100644 index 00000000000..0bcff95af9a --- /dev/null +++ b/cloud/infra/terraform/relay-fence-broker.tf @@ -0,0 +1,243 @@ +locals { + create_relay_fence_broker = local.relay_create_production_ops_identity + relay_fence_state_bucket = "${var.project_id}-terraform-state" + relay_fence_state_prefix = "projects/_/buckets/${local.relay_fence_state_bucket}/objects/terraform/state" +} + +resource "google_service_account" "relay_fence_broker" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-relay-fence" + display_name = "Orca Relay fence broker" + description = "Owns exact reviewed Terraform cell fences behind an authenticated broker." +} + +resource "google_project_iam_custom_role" "relay_fence_broker_mutation" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + role_id = "orcaRelayFenceBroker" + title = "Orca Relay fence broker" + description = "Updates only reviewed Relay MIG sizes and inspects their zone operations." + permissions = [ + "compute.instanceGroupManagers.update" + ] +} + +resource "google_project_iam_member" "relay_fence_broker_mutation" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + role = google_project_iam_custom_role.relay_fence_broker_mutation[0].id + member = google_service_account.relay_fence_broker[0].member +} + +resource "google_project_iam_member" "relay_fence_broker_compute_viewer" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + role = "roles/compute.viewer" + member = google_service_account.relay_fence_broker[0].member +} + +resource "google_project_iam_member" "relay_fence_broker_logging_viewer" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + role = "roles/logging.viewer" + member = google_service_account.relay_fence_broker[0].member +} + +resource "google_project_iam_member" "relay_fence_broker_artifact_reader" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + role = "roles/artifactregistry.reader" + member = google_service_account.relay_fence_broker[0].member +} + +resource "google_storage_bucket_iam_member" "relay_fence_broker_bucket_reader" { + count = local.create_relay_fence_broker ? 1 : 0 + + bucket = local.relay_fence_state_bucket + role = "roles/storage.legacyBucketReader" + member = google_service_account.relay_fence_broker[0].member +} + +resource "google_storage_bucket_iam_member" "relay_fence_broker_state_objects" { + count = local.create_relay_fence_broker ? 1 : 0 + + bucket = local.relay_fence_state_bucket + role = "roles/storage.objectAdmin" + member = google_service_account.relay_fence_broker[0].member + + condition { + title = "relay_fence_exact_objects" + description = "Main state, private saved plans, and the durable broker lease only." + expression = join(" || ", [ + "resource.name.startsWith('${local.relay_fence_state_prefix}/default')", + "resource.name.startsWith('${local.relay_fence_state_prefix}/relay-fence-plans/production/')", + "resource.name.startsWith('${local.relay_fence_state_prefix}/relay-fence-broker/')" + ]) + } +} + +resource "google_service_account_iam_member" "relay_fence_broker_requester_token_creator" { + count = local.create_relay_fence_broker ? 1 : 0 + + service_account_id = google_service_account.github_fence[0].name + role = "roles/iam.serviceAccountTokenCreator" + member = google_service_account.relay_fence_broker[0].member +} + +resource "google_service_account_iam_member" "github_relay_fence_broker_service_account_user" { + count = local.create_relay_fence_broker ? 1 : 0 + + service_account_id = google_service_account.relay_fence_broker[0].name + role = "roles/iam.serviceAccountUser" + member = local.relay_github_deploy_service_account_member +} + +resource "google_cloud_run_v2_service" "relay_fence_broker" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + name = var.relay_fence_broker_service_name + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + deletion_protection = var.environment == "production" + labels = local.relay_shared_labels + + template { + service_account = google_service_account.relay_fence_broker[0].email + timeout = "1800s" + max_instance_request_concurrency = 1 + + scaling { + min_instance_count = 0 + max_instance_count = 1 + } + + containers { + image = var.relay_fence_broker_image + + ports { + container_port = 8080 + } + + env { + name = "ORCA_RELAY_FENCE_PROJECT" + value = var.project_id + } + + env { + name = "ORCA_RELAY_FENCE_STATE_BUCKET" + value = local.relay_fence_state_bucket + } + + env { + name = "ORCA_RELAY_FENCE_LEASE_OBJECT" + value = "terraform/state/relay-fence-broker/production.lock" + } + + env { + name = "ORCA_RELAY_FENCE_DIRECTOR_ORIGIN" + value = var.relay_base_url + } + + env { + name = "ORCA_RELAY_FENCE_ADMIN_AUDIENCE" + value = "${var.relay_base_url}/v1/admin/drain" + } + + env { + name = "ORCA_RELAY_FENCE_REQUESTER_SERVICE_ACCOUNT" + value = google_service_account.github_fence[0].email + } + + env { + name = "ORCA_RELAY_FENCE_RUNTIME_SERVICE_ACCOUNT" + value = google_service_account.relay_runtime.email + } + + env { + name = "ORCA_RELAY_FENCE_SOURCE_CELL_ID" + value = var.relay_fence_source_cell_id + } + + env { + name = "ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID" + value = var.relay_fence_failed_target_cell_id + } + + env { + name = "ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID" + value = var.relay_fence_replacement_target_cell_id + } + + env { + name = "ORCA_RELAY_FENCE_UNOBSERVED_CONNECTION_BOUND" + value = tostring(var.relay_fence_unobserved_connection_bound) + } + + resources { + limits = { + cpu = "1" + memory = "1Gi" + } + + cpu_idle = false + } + + startup_probe { + failure_threshold = 12 + initial_delay_seconds = 0 + period_seconds = 5 + timeout_seconds = 2 + + http_get { + path = "/healthz" + port = 8080 + } + } + } + } + + lifecycle { + ignore_changes = [ + client, + client_version, + template[0].containers[0].image + ] + } + + depends_on = [ + google_project_iam_member.relay_fence_broker_artifact_reader, + google_project_iam_member.relay_fence_broker_compute_viewer, + google_project_iam_member.relay_fence_broker_logging_viewer, + google_project_iam_member.relay_fence_broker_mutation, + google_storage_bucket_iam_member.relay_fence_broker_bucket_reader, + google_storage_bucket_iam_member.relay_fence_broker_state_objects + ] +} + +resource "google_cloud_run_v2_service_iam_member" "relay_fence_broker_invoker" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.relay_fence_broker[0].name + role = "roles/run.invoker" + member = google_service_account.github_fence[0].member +} + +resource "google_cloud_run_v2_service_iam_member" "relay_fence_broker_deploy_invoker" { + count = local.create_relay_fence_broker ? 1 : 0 + + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.relay_fence_broker[0].name + role = "roles/run.invoker" + member = local.relay_github_deploy_service_account_member +} diff --git a/cloud/infra/terraform/relay-gce-cells.tf b/cloud/infra/terraform/relay-gce-cells.tf new file mode 100644 index 00000000000..d6b7f3351f9 --- /dev/null +++ b/cloud/infra/terraform/relay-gce-cells.tf @@ -0,0 +1,393 @@ +locals { + relay_runtime_service_account_email = "${var.name_prefix}-relay@${var.project_id}.iam.gserviceaccount.com" + relay_director_runtime_service_account_email = "${var.name_prefix}-relay-dir@${var.project_id}.iam.gserviceaccount.com" + relay_capacity_service_account_email = var.environment == "production" ? try( + google_service_account.github_production_relay_capacity[0].email, + "" + ) : try(google_service_account.github_staging_relay_capacity[0].email, "") + relay_gce_cells_enabled = local.relay_gce_configured && length(var.relay_gce_cells) > 0 + relay_gce_subnetworks = merge( + { (var.region) = try(google_compute_subnetwork.relay_gce[0].id, null) }, + { for region, subnet in google_compute_subnetwork.relay_gce_additional : region => subnet.id } + ) + relay_gce_topology = { + max_surge = 0 + max_unavailable = 1 + backend_group_count = 1 + public_access_config_count = 0 + backend_timeout_seconds = 86400 + connection_drain_seconds = 300 + } + relay_gce_cell_urls = { + for cell_id, cell in var.relay_gce_cells : + cell_id => "https://${cell.hostname}.${var.relay_gce_domain}" + } + relay_gce_cell_target_sizes = { + for cell_id in keys(var.relay_gce_cells) : + cell_id => contains(var.relay_gce_fenced_cells, cell_id) ? 0 : 1 + } + relay_director_cells = local.relay_gce_cells_enabled ? { + for cell_id, cell in var.relay_gce_cells : cell_id => { + url = local.relay_gce_cell_urls[cell_id] + region = cell.region + capacity_requests = cell.capacity_requests + initially_enabled = cell.initially_enabled + connection_hard_cap = cell.connection_hard_cap + connection_unobserved_bound = cell.connection_unobserved_bound + } + } : { + for cell_id, cell in var.relay_cells : cell_id => { + url = cell.url + region = "us-central1" + capacity_requests = cell.capacity_requests + initially_enabled = true + connection_hard_cap = null + connection_unobserved_bound = null + } + } + relay_director_cells_json = jsonencode([ + for cell_id, cell in local.relay_director_cells : merge( + { + id = cell_id + url = cell.url + region = cell.region + capacityRequests = cell.capacity_requests + initiallyEnabled = cell.initially_enabled + }, + try(cell.connection_hard_cap, null) == null ? {} : { + connectionHardCap = cell.connection_hard_cap + connectionUnobservedBound = cell.connection_unobserved_bound + } + ) + ]) +} + +check "relay_gce_fixed_one_topology" { + assert { + condition = alltrue([ + for region in keys(var.relay_gce_additional_region_subnetwork_cidrs) : region != var.region + ]) && alltrue([ + for cell in values(var.relay_gce_cells) : + cell.region == var.region || contains(keys(var.relay_gce_additional_region_subnetwork_cidrs), cell.region) + ]) + error_message = "Additional Relay regions must differ from the primary region, and every cell region needs a configured subnetwork." + } + + assert { + condition = alltrue([ + for cell_id in var.relay_gce_fenced_cells : contains(keys(var.relay_gce_cells), cell_id) + ]) + error_message = "relay_gce_fenced_cells may contain only configured relay_gce_cells keys." + } + + assert { + condition = alltrue([ + for cell_id in var.relay_region_rehome_source_cell_ids : try( + var.relay_gce_cells[cell_id].region == var.region && + var.relay_gce_cells[cell_id].connection_hard_cap != null && + !contains(var.relay_gce_fenced_cells, cell_id), + false + ) + ]) + error_message = "Regional rehome sources must be configured, unfenced primary-region GCE cells with explicit connection limits." + } + + assert { + condition = length(var.relay_gce_cells) == 0 || var.relay_gce_domain != "" + error_message = "relay_gce_domain is required when GCE cells are configured." + } + + assert { + condition = ( + alltrue([ + for cell_id, target_size in local.relay_gce_cell_target_sizes : + contains(var.relay_gce_fenced_cells, cell_id) ? target_size == 0 : target_size == 1 + ]) && + local.relay_gce_topology.max_surge == 0 && + local.relay_gce_topology.max_unavailable == 1 && + local.relay_gce_topology.backend_group_count == 1 && + local.relay_gce_topology.public_access_config_count == 0 && + local.relay_gce_topology.backend_timeout_seconds == 86400 + ) + error_message = "Relay cells require fixed-one RECREATE MIGs, one non-public backend, and the 86,400-second WebSocket timeout." + } +} + +resource "google_compute_health_check" "relay_gce_liveness" { + count = local.relay_gce_cells_enabled ? 1 : 0 + + project = var.project_id + name = "${local.relay_gce_name}-health" + check_interval_sec = 10 + timeout_sec = 5 + healthy_threshold = 2 + unhealthy_threshold = 3 + + http_health_check { + port = 8080 + request_path = "/health" + } + + log_config { + enable = true + } + + # A project's first Compute API enablement can return before health-check + # creation is accepted, so keep this independent root behind the service. +} + +resource "google_compute_health_check" "relay_gce_readiness" { + count = local.relay_gce_cells_enabled ? 1 : 0 + + project = var.project_id + name = "${local.relay_gce_name}-ready" + check_interval_sec = 10 + timeout_sec = 5 + healthy_threshold = 2 + unhealthy_threshold = 2 + + http_health_check { + port = 8080 + request_path = "/ready" + } + + log_config { + enable = true + } + + # This check has no other Compute dependency to serialize initial API use. +} + +resource "google_compute_instance_template" "relay_gce_cell" { + for_each = var.relay_gce_cells + + project = var.project_id + name_prefix = "${substr("${local.relay_gce_name}-${each.value.hostname}", 0, 52)}-" + machine_type = each.value.machine_type + can_ip_forward = false + tags = ["orca-relay-cell"] + labels = merge( + local.relay_shared_labels, + { + orca-relay-role = "cell" + orca-relay-cell = each.key + }, + each.value.region == var.region ? {} : { orca-relay-region = each.value.region } + ) + + disk { + auto_delete = true + boot = true + device_name = "persistent-disk-0" + disk_size_gb = each.value.boot_disk_gb + disk_type = "pd-balanced" + source_image = each.value.boot_image + } + + network_interface { + subnetwork = local.relay_gce_subnetworks[each.value.region] + + # An empty dynamic block makes the no-public-IP invariant machine-checkable. + dynamic "access_config" { + for_each = range(local.relay_gce_topology.public_access_config_count) + content {} + } + } + + service_account { + email = local.relay_runtime_service_account_email + scopes = ["cloud-platform"] + } + + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + provisioning_model = "STANDARD" + } + + shielded_instance_config { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } + + metadata = { + block-project-ssh-keys = "TRUE" + enable-oslogin = "TRUE" + google-logging-enabled = "TRUE" + } + + metadata_startup_script = templatefile("${path.module}/relay-gce-startup.sh.tftpl", { + project_id = var.project_id + database_secret = google_secret_manager_secret.relay_database_url.secret_id + assignment_secret = google_secret_manager_secret.relay_assignment_signing_key.secret_id + cell_id = each.key + cell_region = each.value.region + include_cell_region = each.value.region != var.region + cell_url = local.relay_gce_cell_urls[each.key] + capacity_requests = each.value.capacity_requests + database_pool_max = each.value.database_pool_max + include_database_pool_max = each.value.region != var.region || each.value.database_pool_max != 10 + connection_hard_cap = each.value.connection_hard_cap + connection_unobserved_bound = each.value.connection_unobserved_bound + auth_issuer = var.auth_base_url + director_url = var.relay_base_url + deploy_service_account = local.relay_github_deploy_service_account_email + capacity_service_account = local.relay_capacity_service_account_email + asia_proof_service_account = local.relay_asia_proof_service_account_email + runtime_service_account = local.relay_runtime_service_account_email + rehome_source_enabled = contains(var.relay_region_rehome_source_cell_ids, each.key) + rehome_director_service_account = local.relay_director_runtime_service_account_email + rehome_audience = "${var.relay_base_url}/v1/admin/host-drain" + artifact_registry_host = "${var.region}-docker.pkg.dev" + relay_image = each.value.image + cloud_sql_proxy_image = var.relay_gce_cloud_sql_proxy_image + # Keep cell-only plans independent from unrelated database configuration drift. + cloud_sql_connection_name = local.relay_database_connection_name + }) + + lifecycle { + create_before_destroy = true + } + + depends_on = [ + google_project_iam_member.relay_runtime_artifact_reader, + google_project_iam_member.relay_runtime_cloudsql_client, + google_project_iam_member.relay_runtime_log_writer, + google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor, + google_secret_manager_secret_iam_member.relay_database_url_accessor + ] +} + +resource "google_compute_instance_group_manager" "relay_gce_cell" { + for_each = var.relay_gce_cells + + project = var.project_id + name = "${local.relay_gce_name}-${each.value.hostname}" + zone = each.value.zone + base_instance_name = "relay-${each.value.hostname}" + target_size = local.relay_gce_cell_target_sizes[each.key] + + version { + name = "primary" + instance_template = google_compute_instance_template.relay_gce_cell[each.key].self_link + } + + named_port { + name = "relay" + port = 8080 + } + + auto_healing_policies { + health_check = google_compute_health_check.relay_gce_liveness[0].id + initial_delay_sec = 180 + } + + update_policy { + type = "PROACTIVE" + minimal_action = "REPLACE" + most_disruptive_allowed_action = "REPLACE" + replacement_method = "RECREATE" + max_surge_fixed = local.relay_gce_topology.max_surge + max_unavailable_fixed = local.relay_gce_topology.max_unavailable + } + + lifecycle { + precondition { + condition = contains([0, 1], local.relay_gce_cell_target_sizes[each.key]) + error_message = "A relay cell MIG must be fenced at zero or active at exactly one." + } + } +} + +resource "google_compute_backend_service" "relay_gce_cell" { + for_each = var.relay_gce_cells + + project = var.project_id + name = "${local.relay_gce_name}-${each.value.hostname}" + protocol = "HTTP" + port_name = "relay" + load_balancing_scheme = "EXTERNAL_MANAGED" + timeout_sec = local.relay_gce_topology.backend_timeout_seconds + connection_draining_timeout_sec = local.relay_gce_topology.connection_drain_seconds + health_checks = [google_compute_health_check.relay_gce_readiness[0].id] + session_affinity = "NONE" + + backend { + group = google_compute_instance_group_manager.relay_gce_cell[each.key].instance_group + balancing_mode = "UTILIZATION" + max_utilization = 0.8 + capacity_scaler = 1 + } + + # Per-connection client IP/status/latency for the data plane; a WebSocket logs once, at close. + log_config { + enable = true + sample_rate = var.relay_gce_cell_log_sample_rate + } + + lifecycle { + precondition { + condition = local.relay_gce_topology.backend_group_count == 1 + error_message = "Each exact relay host must route to one non-overlapping fixed-one MIG." + } + } +} + +resource "google_compute_url_map" "relay_gce" { + count = local.relay_gce_cells_enabled ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + default_service = google_compute_backend_service.relay_gce_cell[sort(keys(var.relay_gce_cells))[0]].id + + # Unknown wildcard hosts fail at the LB and can never fall through to a cell. + default_route_action { + fault_injection_policy { + abort { + http_status = 404 + percentage = 100 + } + } + } + + dynamic "host_rule" { + for_each = var.relay_gce_cells + iterator = cell + content { + hosts = ["${cell.value.hostname}.${var.relay_gce_domain}"] + path_matcher = "cell-${cell.value.hostname}" + } + } + + dynamic "path_matcher" { + for_each = var.relay_gce_cells + iterator = cell + content { + name = "cell-${cell.value.hostname}" + default_service = google_compute_backend_service.relay_gce_cell[cell.key].id + } + } +} + +resource "google_compute_target_https_proxy" "relay_gce" { + count = local.relay_gce_cells_enabled ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + url_map = google_compute_url_map.relay_gce[0].id + certificate_map = "//certificatemanager.googleapis.com/${google_certificate_manager_certificate_map.relay_gce[0].id}" + quic_override = "NONE" +} + +resource "google_compute_global_forwarding_rule" "relay_gce" { + count = local.relay_gce_cells_enabled ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + ip_address = google_compute_global_address.relay_gce[0].id + port_range = "443" + target = google_compute_target_https_proxy.relay_gce[0].id + load_balancing_scheme = "EXTERNAL_MANAGED" + network_tier = "PREMIUM" +} diff --git a/cloud/infra/terraform/relay-gce-foundation.tf b/cloud/infra/terraform/relay-gce-foundation.tf new file mode 100644 index 00000000000..aab3b4579eb --- /dev/null +++ b/cloud/infra/terraform/relay-gce-foundation.tf @@ -0,0 +1,200 @@ +locals { + relay_gce_configured = var.relay_gce_domain != "" + relay_gce_name = "${var.name_prefix}-relay-gce" +} + +resource "google_compute_network" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + auto_create_subnetworks = false + routing_mode = "REGIONAL" +} + +resource "google_compute_subnetwork" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + region = var.region + network = google_compute_network.relay_gce[0].id + ip_cidr_range = var.relay_gce_subnetwork_cidr + private_ip_google_access = true + stack_type = "IPV4_ONLY" +} + +resource "google_compute_router" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + region = var.region + network = google_compute_network.relay_gce[0].id +} + +resource "google_compute_router_nat" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + region = var.region + router = google_compute_router.relay_gce[0].name + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" + + subnetwork { + name = google_compute_subnetwork.relay_gce[0].id + source_ip_ranges_to_nat = ["ALL_IP_RANGES"] + } + + log_config { + enable = true + filter = "ERRORS_ONLY" + } +} + +# The primary US resources above retain their production addresses. New regions are additive. +resource "google_compute_subnetwork" "relay_gce_additional" { + for_each = local.relay_gce_configured ? var.relay_gce_additional_region_subnetwork_cidrs : {} + + project = var.project_id + name = "${local.relay_gce_name}-${each.key}" + region = each.key + network = google_compute_network.relay_gce[0].id + ip_cidr_range = each.value + private_ip_google_access = true + stack_type = "IPV4_ONLY" +} + +resource "google_compute_router" "relay_gce_additional" { + for_each = local.relay_gce_configured ? var.relay_gce_additional_region_subnetwork_cidrs : {} + + project = var.project_id + name = "${local.relay_gce_name}-${each.key}" + region = each.key + network = google_compute_network.relay_gce[0].id +} + +resource "google_compute_router_nat" "relay_gce_additional" { + for_each = local.relay_gce_configured ? var.relay_gce_additional_region_subnetwork_cidrs : {} + + project = var.project_id + name = "${local.relay_gce_name}-${each.key}" + region = each.key + router = google_compute_router.relay_gce_additional[each.key].name + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" + + subnetwork { + name = google_compute_subnetwork.relay_gce_additional[each.key].id + source_ip_ranges_to_nat = ["ALL_IP_RANGES"] + } + + log_config { + enable = true + filter = "ERRORS_ONLY" + } +} + +resource "google_compute_firewall" "relay_gce_load_balancer" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = "${local.relay_gce_name}-lb" + network = google_compute_network.relay_gce[0].name + direction = "INGRESS" + source_ranges = ["35.191.0.0/16", "130.211.0.0/22"] + target_tags = ["orca-relay-cell"] + + allow { + protocol = "tcp" + ports = ["8080"] + } +} + +resource "google_compute_firewall" "relay_gce_iap_ssh" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = "${local.relay_gce_name}-iap-ssh" + network = google_compute_network.relay_gce[0].name + direction = "INGRESS" + source_ranges = ["35.235.240.0/20"] + target_tags = ["orca-relay-cell"] + + allow { + protocol = "tcp" + ports = ["22"] + } +} + +resource "google_project_iam_member" "relay_runtime_artifact_reader" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + role = "roles/artifactregistry.reader" + member = google_service_account.relay_runtime.member +} + +resource "google_project_iam_member" "relay_runtime_log_writer" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + role = "roles/logging.logWriter" + member = google_service_account.relay_runtime.member +} + +resource "google_compute_global_address" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + address_type = "EXTERNAL" + ip_version = "IPV4" +} + +resource "google_certificate_manager_dns_authorization" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + domain = var.relay_gce_domain + location = "global" + type = "PER_PROJECT_RECORD" + description = "DNS authorization for Orca Relay wildcard cell certificates." + labels = local.relay_shared_labels +} + +resource "google_certificate_manager_certificate" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + location = "global" + description = "Wildcard certificate for exact-routed Orca Relay GCE cells." + labels = local.relay_shared_labels + + managed { + domains = ["*.${var.relay_gce_domain}"] + dns_authorizations = [google_certificate_manager_dns_authorization.relay_gce[0].id] + } +} + +resource "google_certificate_manager_certificate_map" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = local.relay_gce_name + description = "Certificate map for the shared Orca Relay HTTPS load balancer." +} + +resource "google_certificate_manager_certificate_map_entry" "relay_gce" { + count = local.relay_gce_configured ? 1 : 0 + + project = var.project_id + name = "${local.relay_gce_name}-wildcard" + map = google_certificate_manager_certificate_map.relay_gce[0].name + hostname = "*.${var.relay_gce_domain}" + certificates = [google_certificate_manager_certificate.relay_gce[0].id] +} diff --git a/cloud/infra/terraform/relay-gce-startup.sh.tftpl b/cloud/infra/terraform/relay-gce-startup.sh.tftpl new file mode 100644 index 00000000000..f593d94e9e5 --- /dev/null +++ b/cloud/infra/terraform/relay-gce-startup.sh.tftpl @@ -0,0 +1,136 @@ +#!/bin/bash +set -euo pipefail + +readonly metadata_url="http://metadata.google.internal/computeMetadata/v1" +readonly state_dir="/var/lib/orca-relay" +readonly cloudsql_dir="$${state_dir}/cloudsql" +readonly docker_config_dir="$${state_dir}/docker" +readonly env_file="$${state_dir}/relay.env" + +# COS mounts /root read-only, and neither the plaintext environment nor pull credential should survive bootstrap. +trap 'rm -f "$${env_file}"; rm -rf "$${docker_config_dir}"' EXIT + +mkdir -p "$${cloudsql_dir}" "$${docker_config_dir}" +chmod 0700 "$${state_dir}" +chmod 0700 "$${docker_config_dir}" +chmod 0777 "$${cloudsql_dir}" +export DOCKER_CONFIG="$${docker_config_dir}" + +# Startup metadata can run before COS has made the Docker socket usable. +systemctl start docker +for _ in $(seq 1 60); do + if docker info >/dev/null 2>&1; then + break + fi + sleep 1 +done +docker info >/dev/null + +metadata_access_token() { + curl --fail --silent --show-error \ + --header 'Metadata-Flavor: Google' \ + "$${metadata_url}/instance/service-accounts/default/token" \ + | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p' +} + +read_secret() { + local secret_name="$1" + local token="$2" + local encoded + encoded="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $${token}" \ + "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$${secret_name}/versions/latest:access" \ + | sed -n 's/.*"data":[[:space:]]*"\([^"]*\)".*/\1/p')" + test -n "$${encoded}" + printf '%s' "$${encoded}" | tr '_-' '/+' | base64 --decode +} + +access_token="$(metadata_access_token)" +test -n "$${access_token}" +database_url="$(read_secret '${database_secret}' "$${access_token}")" +assignment_key="$(read_secret '${assignment_secret}' "$${access_token}")" + +umask 077 +{ + printf 'DATABASE_URL=%s\n' "$${database_url}" + printf 'ORCA_RELAY_ASSIGNMENT_SIGNING_KEY=%s\n' "$${assignment_key}" + printf 'ORCA_RELAY_PUBLIC_URL=%s\n' '${cell_url}' + printf 'ORCA_RELAY_CELL_URL=%s\n' '${cell_url}' + printf 'ORCA_RELAY_AUTH_ISSUER=%s\n' '${auth_issuer}' + printf 'ORCA_RELAY_AUTH_AUDIENCE=orca-relay\n' + printf 'ORCA_RELAY_JWKS_URL=%s/.well-known/jwks.json\n' '${auth_issuer}' + printf 'ORCA_RELAY_ROLE=cell\n' + printf 'ORCA_RELAY_CELL_ID=%s\n' '${cell_id}' +%{ if include_cell_region ~} + printf 'ORCA_RELAY_REGION=%s\n' '${cell_region}' +%{ endif ~} + printf 'ORCA_RELAY_CELL_CAPACITY=%s\n' '${capacity_requests}' +%{ if include_database_pool_max ~} + printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\n' '${database_pool_max}' +%{ endif ~} +%{ if connection_hard_cap != null ~} + printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\n' '${connection_hard_cap}' + printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\n' '${connection_unobserved_bound}' +%{ endif ~} + printf 'ORCA_RELAY_CELLS_JSON=[]\n' + printf 'ORCA_RELAY_ADMIN_AUDIENCE=%s/v1/admin/drain\n' '${director_url}' + printf 'ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT=%s\n' '${deploy_service_account}' + printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\n' '${capacity_service_account}' +%{ if asia_proof_service_account != "" ~} + printf 'ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT=%s\n' '${asia_proof_service_account}' +%{ endif ~} + printf 'ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT=%s\n' '${runtime_service_account}' +%{ if rehome_source_enabled ~} + printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\n' '${rehome_director_service_account}' + printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\n' '${rehome_audience}' +%{ endif ~} + printf 'ORCA_RELAY_DIRECTOR_URL=%s\n' '${director_url}' + printf 'ORCA_RELAY_HEARTBEAT_AUDIENCE=%s/v1/admin/cell-heartbeat\n' '${director_url}' + printf 'ORCA_RELAY_IMAGE_DIGEST=%s\n' '${trimprefix(regex("@sha256:[a-f0-9]{64}$", relay_image), "@")}' +} > "$${env_file}" +unset database_url assignment_key + +# COS has no long-lived registry credential; use a short metadata token only for the pull. +printf '%s' "$${access_token}" \ + | docker login --username oauth2accesstoken --password-stdin 'https://${artifact_registry_host}' +docker pull '${relay_image}' +docker logout '${artifact_registry_host}' >/dev/null 2>&1 || true +unset access_token +docker pull '${cloud_sql_proxy_image}' + +docker rm --force orca-relay cloud-sql-proxy >/dev/null 2>&1 || true +# The persistent COS state directory can retain a dead proxy's socket across VM reboots. +find "$${cloudsql_dir}" -type s -name '.s.PGSQL.5432' -delete +docker run --detach \ + --name cloud-sql-proxy \ + --restart always \ + --security-opt no-new-privileges \ + --cap-drop ALL \ + --user 0:0 \ + --volume "$${cloudsql_dir}:/cloudsql" \ + '${cloud_sql_proxy_image}' \ + --unix-socket=/cloudsql \ + '${cloud_sql_connection_name}' + +# Readiness depends on the proxy socket, so do not start the relay into a known SQL failure. +for _ in $(seq 1 60); do + if find "$${cloudsql_dir}" -type s -name '.s.PGSQL.5432' -print -quit | grep -q .; then + break + fi + sleep 1 +done +find "$${cloudsql_dir}" -type s -name '.s.PGSQL.5432' -print -quit | grep -q . + +docker run --detach \ + --name orca-relay \ + --restart always \ + --stop-timeout 300 \ + --security-opt no-new-privileges \ + --cap-drop ALL \ + --publish 8080:8080 \ + --volume "$${cloudsql_dir}:/cloudsql" \ + --env-file "$${env_file}" \ + '${relay_image}' + +# GCE cells feed the same privacy-safe aggregate metrics as Cloud Run without app credentials. +systemctl start logging-agent.target diff --git a/cloud/infra/terraform/relay-github-actions.tf b/cloud/infra/terraform/relay-github-actions.tf new file mode 100644 index 00000000000..450ea64cc0a --- /dev/null +++ b/cloud/infra/terraform/relay-github-actions.tf @@ -0,0 +1,669 @@ +# GitHub Actions identities the relay root owns. +# +# The shared deploy account, its Workload Identity provider, and everything bound to it are +# environment-conditional under amendment A1: production is relay-owned, staging is apps-owned. +# Both state surgeries are done, so their counts are production-only here; the staging copies +# are declared by infra/terraform-apps and live in its state. +# +# The Workload Identity pool itself is foundation-owned and reached by literal through +# relay-shared.tf, never by reference. + +locals { + github_monitor_caller_workflow_file = "monitor-relay-production.yml" + github_monitor_workflow_file = "monitor-relay-production-job.yml" + github_fence_workflow_file = "deploy-relay-production-multi-target.yml" + github_production_relay_workflow_files = [ + "deploy-relay-fence-broker.yml", + "deploy-relay-production-capacity.yml", + "deploy-relay-production-director.yml", + "deploy-relay-production-multi-target.yml", + "deploy-relay-production.yml", + "operate-relay-asia-admission.yml", + "publish-relay-production.yml" + ] + github_production_relay_capacity_workflow_file = "deploy-relay-production-capacity.yml" + github_production_relay_capacity_job_workflow_file = "deploy-relay-production-capacity-job.yml" + github_production_relay_same_cap_workflow_file = "deploy-relay-production-same-cap.yml" + github_production_relay_same_cap_job_workflow_file = "deploy-relay-production-same-cap-job.yml" + github_production_relay_rehome_workflow_file = "operate-relay-production-rehome.yml" + github_production_relay_rehome_job_workflow_file = "operate-relay-production-rehome-job.yml" + github_staging_relay_capacity_workflow_files = [ + "bootstrap-relay-staging-capacity.yml", + "prove-relay-staging-capacity.yml", + "recover-relay-staging-c4-image.yml" + ] + + # The same-cap caller admits its own jobs too: release_lease runs in the caller file and presents + # the caller as job_workflow_ref, so pinning only the reusable job would refuse it. + github_production_relay_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "((${join(" || ", [for workflow_file in local.github_production_relay_workflow_files : "assertion.workflow_ref == '${prefix}${workflow_file}@refs/heads/main'"])}) || (assertion.workflow_ref == '${prefix}${local.github_production_relay_rehome_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_production_relay_rehome_job_workflow_file}@refs/heads/main') || (assertion.workflow_ref == '${prefix}${local.github_production_relay_same_cap_workflow_file}@refs/heads/main' && (assertion.job_workflow_ref == '${prefix}${local.github_production_relay_same_cap_job_workflow_file}@refs/heads/main' || assertion.job_workflow_ref == '${prefix}${local.github_production_relay_same_cap_workflow_file}@refs/heads/main')))" + ] + github_monitor_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "assertion.workflow_ref == '${prefix}${local.github_monitor_caller_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_monitor_workflow_file}@refs/heads/main'" + ] + github_fence_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "assertion.workflow_ref == '${prefix}${local.github_fence_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_fence_workflow_file}@refs/heads/main'" + ] + github_production_relay_capacity_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "((assertion.workflow_ref == '${prefix}${local.github_production_relay_capacity_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_production_relay_capacity_job_workflow_file}@refs/heads/main') || (assertion.workflow_ref == '${prefix}${local.github_production_relay_same_cap_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_production_relay_same_cap_job_workflow_file}@refs/heads/main'))" + ] + github_staging_relay_capacity_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "(${join(" || ", [for workflow_file in local.github_staging_relay_capacity_workflow_files : "assertion.workflow_ref == '${prefix}${workflow_file}@refs/heads/main'"])})" + ] + + create_staging_relay_power_role = ( + local.relay_create_github_deploy_identity && var.environment == "staging" + ) + create_staging_relay_capacity_identity = ( + local.relay_create_github_deploy_identity && var.environment == "staging" + ) + create_production_relay_capacity_identity = ( + local.relay_create_github_deploy_identity && var.environment == "production" + ) +} + +resource "google_iam_workload_identity_pool_provider" "github" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github" + display_name = "GitHub" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.actor" = "assertion.actor" + "attribute.environment" = "assertion.environment" + "attribute.ref" = "assertion.ref" + "attribute.repository" = "assertion.repository" + "attribute.repository_id" = "assertion.repository_id" + "attribute.repository_owner_id" = "assertion.repository_owner_id" + "attribute.workflow_ref" = "assertion.workflow_ref" + } + + # Production-only, so the condition is unconditional. The staging copy of this provider is + # declared by infra/terraform-apps/github-actions.tf and pinned there. + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'production'", + local.relay_github_workflow_conditions["github"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_iam_workload_identity_pool_provider" "github_monitor" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-relay-monitor" + display_name = "GitHub Relay production monitor" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.ref" = "assertion.ref" + "attribute.environment" = "assertion.environment" + "attribute.job_workflow_ref" = "assertion.job_workflow_ref" + "attribute.relay_ops_identity" = "'monitor'" + } + + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'production'", + local.relay_github_workflow_conditions["github_monitor"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_iam_workload_identity_pool_provider" "github_fence" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-relay-fence" + display_name = "GitHub Relay production fence" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.ref" = "assertion.ref" + "attribute.environment" = "assertion.environment" + "attribute.job_workflow_ref" = "assertion.job_workflow_ref" + "attribute.relay_ops_identity" = "'fence'" + } + + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'production'", + local.relay_github_workflow_conditions["github_fence"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_iam_workload_identity_pool_provider" "github_staging_relay_capacity" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-relay-capacity" + display_name = "GitHub Relay staging capacity" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.ref" = "assertion.ref" + "attribute.environment" = "assertion.environment" + "attribute.workflow_ref" = "assertion.workflow_ref" + "attribute.relay_ops_identity" = "'staging-capacity'" + } + + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'staging'", + local.relay_github_workflow_conditions["github_staging_relay_capacity"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_iam_workload_identity_pool_provider" "github_production_relay_capacity" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-relay-capacity" + display_name = "GitHub Relay production capacity" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.ref" = "assertion.ref" + "attribute.environment" = "assertion.environment" + "attribute.workflow_ref" = "assertion.workflow_ref" + "attribute.job_workflow_ref" = "assertion.job_workflow_ref" + "attribute.relay_ops_identity" = "'production-capacity'" + } + + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'production'", + local.relay_github_workflow_conditions["github_production_relay_capacity"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account" "github_deploy" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-deploy" + display_name = "Orca Cloud GitHub deploy" + description = "Deploys Orca Cloud from GitHub Actions." +} + +resource "google_service_account" "github_monitor" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-monitor" + display_name = "Orca Relay production monitor" + description = "Reads aggregate Relay production telemetry." +} + +resource "google_service_account" "github_fence" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-fence" + display_name = "Orca Relay production fence requester" + description = "Requests exact reviewed Relay cell fences through the private broker." +} + +resource "google_service_account" "github_staging_relay_capacity" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-cap" + display_name = "Orca Relay staging capacity transition" + description = "Runs the exact reviewed Relay staging capacity workflow." +} + +resource "google_service_account" "github_production_relay_capacity" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-cap" + display_name = "Orca Relay production capacity transition" + description = "Runs the exact reviewed Relay production capacity workflow." +} + +resource "google_service_account_iam_member" "github_workload_identity_user" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + service_account_id = google_service_account.github_deploy[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.repository/${local.relay_github_repository}" +} + +# The `github` provider maps assertion.repository straight through, so the binding above admits +# only the primary repository. Each additional accepted repository needs its own principalSet +# before its workflows can mint this account; with an empty list this creates nothing. +resource "google_service_account_iam_member" "github_accepted_repository_workload_identity_user" { + for_each = toset( + local.relay_create_production_ops_identity + ? slice(local.relay_github_accepted_repository_names, 1, length(local.relay_github_accepted_repository_names)) + : [] + ) + + service_account_id = google_service_account.github_deploy[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.repository/${each.key}" +} + +resource "google_service_account_iam_member" "github_monitor_workload_identity_user" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + service_account_id = google_service_account.github_monitor[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/monitor" +} + +resource "google_service_account_iam_member" "github_fence_workload_identity_user" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + service_account_id = google_service_account.github_fence[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/fence" +} + +resource "google_service_account_iam_member" "github_staging_relay_capacity_workload_identity_user" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + service_account_id = google_service_account.github_staging_relay_capacity[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/staging-capacity" +} + +resource "google_service_account_iam_member" "github_production_relay_capacity_workload_identity_user" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + service_account_id = google_service_account.github_production_relay_capacity[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/production-capacity" +} + +resource "google_artifact_registry_repository_iam_member" "github_production_relay_writer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + location = var.region + repository = var.artifact_repository_id + role = "roles/artifactregistry.writer" + member = local.relay_github_deploy_service_account_member +} + +resource "google_artifact_registry_repository_iam_member" "github_production_relay_staging_mirror_writer" { + count = local.relay_create_github_deploy_identity && var.environment == "staging" ? 1 : 0 + + project = var.project_id + location = var.region + repository = var.artifact_repository_id + role = "roles/artifactregistry.writer" + member = "serviceAccount:orca-cloud-gha-deploy@onorca-cloud.iam.gserviceaccount.com" +} + +resource "google_cloud_run_v2_service_iam_member" "github_production_relay_director_developer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + location = var.region + name = var.relay_cloud_run_service_name + role = "roles/run.developer" + member = local.relay_github_deploy_service_account_member +} + +resource "google_cloud_run_v2_service_iam_member" "github_production_relay_fence_broker_developer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + location = var.region + name = var.relay_fence_broker_service_name + role = "roles/run.developer" + member = local.relay_github_deploy_service_account_member +} + +# Candidate preflight reads GCE/LB topology; Terraform fencing keeps mutation narrowly scoped. +resource "google_project_iam_member" "github_compute_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/compute.viewer" + member = google_service_account.github_deploy[0].member +} + +# Incident monitoring needs aggregate telemetry and inventory without mutation permissions. +resource "google_project_iam_member" "github_monitoring_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/monitoring.viewer" + member = google_service_account.github_deploy[0].member +} + +resource "google_project_iam_member" "github_logging_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/logging.viewer" + member = google_service_account.github_deploy[0].member +} + +resource "google_project_iam_member" "github_cloudsql_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/cloudsql.viewer" + member = google_service_account.github_deploy[0].member +} + +resource "google_project_iam_member" "github_relay_monitor_monitoring_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/monitoring.viewer" + member = google_service_account.github_monitor[0].member +} + +resource "google_project_iam_member" "github_relay_monitor_logging_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/logging.viewer" + member = google_service_account.github_monitor[0].member +} + +resource "google_project_iam_member" "github_relay_monitor_cloudsql_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/cloudsql.viewer" + member = google_service_account.github_monitor[0].member +} + +resource "google_project_iam_member" "github_monitor_compute_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/compute.viewer" + member = google_service_account.github_monitor[0].member +} + +resource "google_project_iam_member" "github_fence_monitoring_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/monitoring.viewer" + member = google_service_account.github_fence[0].member +} + +resource "google_project_iam_member" "github_fence_logging_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/logging.viewer" + member = google_service_account.github_fence[0].member +} + +resource "google_project_iam_member" "github_fence_cloudsql_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/cloudsql.viewer" + member = google_service_account.github_fence[0].member +} + +resource "google_project_iam_member" "github_fence_compute_viewer" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + project = var.project_id + role = "roles/compute.viewer" + member = google_service_account.github_fence[0].member +} + +# Staging may sleep between test windows. Keep its workflow narrower than a +# general Compute/Cloud SQL editor and never create this role in production. +resource "google_project_iam_custom_role" "github_staging_relay_power" { + count = local.create_staging_relay_power_role ? 1 : 0 + + project = var.project_id + role_id = "orcaRelayStagingPower" + title = "Orca Relay staging power operator" + description = "Scales staging Relay MIGs and starts or stops its shared staging database." + permissions = [ + "cloudsql.instances.get", + "cloudsql.instances.update", + "cloudsql.operations.get", + "compute.instanceGroupManagers.get", + "compute.instanceGroupManagers.update", + "compute.zoneOperations.get" + ] +} + +resource "google_project_iam_member" "github_staging_relay_power" { + count = local.create_staging_relay_power_role ? 1 : 0 + + project = var.project_id + role = google_project_iam_custom_role.github_staging_relay_power[0].id + member = local.relay_github_deploy_service_account_member +} + +resource "google_project_iam_custom_role" "github_staging_relay_capacity_mutation" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role_id = "orcaRelayStagingCapacity" + title = "Orca Relay staging capacity transition" + description = "Replaces one staging Relay template and restarts its managed instance group." + permissions = [ + "compute.disks.create", + "compute.healthChecks.use", + "compute.images.useReadOnly", + "compute.instanceGroupManagers.get", + "compute.instanceGroupManagers.update", + "compute.instances.create", + "compute.instances.setLabels", + "compute.instances.setMetadata", + "compute.instances.setTags", + "compute.instanceTemplates.create", + "compute.instanceTemplates.delete", + "compute.instanceTemplates.get", + "compute.instanceTemplates.useReadOnly", + "compute.networks.use", + "compute.subnetworks.use", + "compute.zoneOperations.get" + ] +} + +resource "google_project_iam_member" "github_staging_relay_capacity_mutation" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role = google_project_iam_custom_role.github_staging_relay_capacity_mutation[0].id + member = google_service_account.github_staging_relay_capacity[0].member +} + +resource "google_project_iam_member" "github_staging_relay_capacity_viewer" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role = "roles/viewer" + member = google_service_account.github_staging_relay_capacity[0].member +} + +resource "google_project_iam_member" "github_staging_relay_capacity_artifact_reader" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role = "roles/artifactregistry.reader" + member = google_service_account.github_staging_relay_capacity[0].member +} + +resource "google_cloud_run_v2_service_iam_member" "github_staging_relay_capacity_developer" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + project = var.project_id + location = var.region + name = var.relay_cloud_run_service_name + role = "roles/run.developer" + member = google_service_account.github_staging_relay_capacity[0].member +} + +resource "google_storage_bucket_iam_member" "github_staging_relay_capacity_state" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + bucket = "${var.project_id}-terraform-state" + role = "roles/storage.objectAdmin" + member = google_service_account.github_staging_relay_capacity[0].member + + condition { + title = "relay_capacity_state" + description = "Limits the staging capacity workflow to the default Terraform state and lock." + expression = join(" || ", [ + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" + ]) + } +} + +resource "google_service_account_iam_member" "github_staging_relay_capacity_runtime_user" { + count = local.create_staging_relay_capacity_identity ? 1 : 0 + + service_account_id = google_service_account.relay_runtime.name + role = "roles/iam.serviceAccountUser" + member = google_service_account.github_staging_relay_capacity[0].member +} + +resource "google_project_iam_custom_role" "github_production_relay_capacity_mutation" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role_id = "orcaRelayProductionCapacity" + title = "Orca Relay production capacity transition" + description = "Replaces exactly one production Relay template and its managed instance." + permissions = [ + "compute.disks.create", + "compute.healthChecks.use", + "compute.images.useReadOnly", + "compute.instanceGroupManagers.get", + "compute.instanceGroupManagers.update", + "compute.instances.create", + "compute.instances.setLabels", + "compute.instances.setMetadata", + "compute.instances.setTags", + "compute.instanceTemplates.create", + "compute.instanceTemplates.delete", + "compute.instanceTemplates.get", + "compute.instanceTemplates.useReadOnly", + "compute.networks.use", + "compute.subnetworks.use", + "compute.zoneOperations.get" + ] +} + +resource "google_project_iam_member" "github_production_relay_capacity_mutation" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role = google_project_iam_custom_role.github_production_relay_capacity_mutation[0].id + member = google_service_account.github_production_relay_capacity[0].member +} + +resource "google_project_iam_member" "github_production_relay_capacity_viewer" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role = "roles/viewer" + member = google_service_account.github_production_relay_capacity[0].member +} + +resource "google_project_iam_member" "github_production_relay_capacity_artifact_reader" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + project = var.project_id + role = "roles/artifactregistry.reader" + member = google_service_account.github_production_relay_capacity[0].member +} + +resource "google_storage_bucket_iam_member" "github_production_relay_capacity_state" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + bucket = "${var.project_id}-terraform-state" + role = "roles/storage.objectAdmin" + member = google_service_account.github_production_relay_capacity[0].member + + condition { + title = "relay_production_capacity_state" + description = "Limits the production capacity workflow to the default Terraform state and lock." + expression = join(" || ", [ + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" + ]) + } +} + +resource "google_service_account_iam_member" "github_production_relay_capacity_runtime_user" { + count = local.create_production_relay_capacity_identity ? 1 : 0 + + service_account_id = google_service_account.relay_runtime.name + role = "roles/iam.serviceAccountUser" + member = google_service_account.github_production_relay_capacity[0].member +} + +# Candidate preflight reads reviewed Terraform outputs, but must not be able to mutate state. +resource "google_storage_bucket_iam_member" "github_terraform_state_reader" { + count = local.relay_create_production_ops_identity ? 1 : 0 + + bucket = "${var.project_id}-terraform-state" + role = "roles/storage.objectViewer" + member = google_service_account.github_deploy[0].member +} + + +# Relay deploys replace only the image while retaining the Terraform-owned +# runtime identity and service shape. +resource "google_service_account_iam_member" "github_relay_runtime_service_account_user" { + count = local.relay_create_github_deploy_identity ? 1 : 0 + + service_account_id = google_service_account.relay_runtime.name + role = "roles/iam.serviceAccountUser" + member = local.relay_github_deploy_service_account_member +} + +resource "google_service_account_iam_member" "github_relay_director_runtime_service_account_user" { + count = local.relay_create_github_deploy_identity ? 1 : 0 + + service_account_id = google_service_account.relay_director_runtime.name + role = "roles/iam.serviceAccountUser" + member = local.relay_github_deploy_service_account_member +} + diff --git a/cloud/infra/terraform/relay-github-workflow-trust.tf b/cloud/infra/terraform/relay-github-workflow-trust.tf new file mode 100644 index 00000000000..9cf57f05198 --- /dev/null +++ b/cloud/infra/terraform/relay-github-workflow-trust.tf @@ -0,0 +1,38 @@ +# The workflow half of every relay Workload Identity condition, one clause per accepted +# repository. +# +# Each provider file builds its own clause list from local.relay_github_workflow_ref_prefixes, so +# a workflow allowlist stays next to the identity it authorizes. This file only names those lists +# and renders them: with a single accepted repository the clause is spliced in unchanged, and with +# more it becomes one parenthesised OR arm per repository, each arm carrying its own repository +# claims. The repository-independent claims (ref, environment, event_name) stay outside the OR in +# the provider blocks. +locals { + relay_github_workflow_clauses = { + github = local.github_production_relay_workflow_clauses + github_monitor = local.github_monitor_workflow_clauses + github_fence = local.github_fence_workflow_clauses + github_production_relay_capacity = local.github_production_relay_capacity_workflow_clauses + github_staging_relay_capacity = local.github_staging_relay_capacity_workflow_clauses + github_staging_relay_deploy = local.github_staging_relay_deploy_workflow_clauses + github_relay_asia_topology = local.github_relay_asia_topology_workflow_clauses + github_relay_asia_proof = local.github_relay_asia_proof_workflow_clauses + } + + relay_github_workflow_arms = { + for name, clauses in local.relay_github_workflow_clauses : + name => [ + for index, clause in clauses : + "(${local.relay_github_accepted_repository_claims[index]} && ${clause})" + ] + } + + relay_github_workflow_conditions = { + for name, clauses in local.relay_github_workflow_clauses : + name => ( + local.relay_github_single_repository + ? clauses[0] + : "(${join(" || ", local.relay_github_workflow_arms[name])})" + ) + } +} diff --git a/cloud/infra/terraform/relay-observability.tf b/cloud/infra/terraform/relay-observability.tf new file mode 100644 index 00000000000..a8fa4276eb2 --- /dev/null +++ b/cloud/infra/terraform/relay-observability.tf @@ -0,0 +1,525 @@ +locals { + relay_service_names = concat( + [var.relay_cloud_run_service_name], + [for cell in values(var.relay_cells) : cell.service_name] + ) + relay_service_log_filter = join(" OR ", [ + for name in local.relay_service_names : "resource.labels.service_name=\"${name}\"" + ]) + relay_runtime_log_filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR (resource.type=\"gce_instance\" AND jsonPayload.role=\"cell\")) AND jsonPayload.event=\"orca_relay_runtime_metrics\"" + relay_gce_connection_warning_thresholds = { + for cell_id, cell in var.relay_gce_cells : + cell_id => cell.connection_hard_cap == null ? 550 : floor( + (cell.connection_hard_cap - 100 - cell.connection_unobserved_bound) * 0.85 + ) + } + relay_gce_connection_warning_groups = { + for threshold in distinct(values(local.relay_gce_connection_warning_thresholds)) : + tostring(threshold) => sort([ + for cell_id, cell_threshold in local.relay_gce_connection_warning_thresholds : + cell_id if cell_threshold == threshold + ]) + } + relay_incident_metrics = { + assignment_5xx = { + description = "Director assignment requests returning a server error." + filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND httpRequest.requestMethod=\"POST\" AND httpRequest.requestUrl=~\"/v1/assign$\" AND httpRequest.status>=500" + } + assignment_edge_429 = { + description = "Director assignment or resolve requests rejected before an instance was available." + filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND httpRequest.requestMethod=\"POST\" AND httpRequest.requestUrl=~\"/v1/(assign|resolve)$\" AND httpRequest.status=429" + } + postgres_retries = { + description = "Relay PostgreSQL transactions recovered after a retryable abort." + filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_postgres_transaction_retry\"" + } + postgres_retry_exhausted = { + description = "Relay PostgreSQL transactions that exhausted bounded retry." + filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_postgres_transaction_exhausted\"" + } + } + + relay_runtime_metrics = { + total_connections = { field = "totalConnections", description = "Open relay WebSocket requests per process." } + controls = { field = "controls", description = "Authenticated standing desktop controls per process." } + splices = { field = "splices", description = "Active phone-to-desktop ciphertext splices per process." } + pending_splices = { field = "pendingSplices", description = "Phone connections waiting for their host data leg." } + queued_bytes = { field = "queuedBytes", description = "Process-wide relay backpressure bytes." } + http_latency_ms = { field = "httpLatencyMsMax", description = "Maximum director/administrative HTTP latency in the interval." } + sql_latency_ms = { field = "sqlLatencyMsMax", description = "Maximum observed SQL operation latency in the interval." } + control_renewal_latency_ms_p50 = { field = "controlRenewalLatencyMsP50", description = "Control renewal latency p50 in the interval." } + control_renewal_latency_ms_p95 = { field = "controlRenewalLatencyMsP95", description = "Control renewal latency p95 in the interval." } + control_renewal_latency_ms_max = { field = "controlRenewalLatencyMsMax", description = "Maximum control renewal latency in the interval." } + control_renewals = { field = "controlRenewalsDelta", description = "Control renewal attempts in the interval." } + control_renewal_successes = { field = "controlRenewalSuccessesDelta", description = "Successful control renewals in the interval." } + control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." } + control_activity_recoveries = { field = "controlActivityRecoveriesDelta", description = "Control activity leases recovered after a renewal miss." } + control_activity_recovery_failures = { field = "controlActivityRecoveryFailuresDelta", description = "Control activity lease recovery attempts that failed." } + heap_used_bytes = { field = "heapUsedBytes", description = "Node.js heap bytes used by the relay process." } + event_loop_ms_p99 = { field = "eventLoopDelayMsP99", description = "Node.js event-loop delay p99 in milliseconds." } + forwarded_bytes = { field = "forwardedBytesDelta", description = "Ciphertext bytes admitted for forwarding." } + auth_successes = { field = "authSuccessesDelta", description = "Successful outer relay authentication stages." } + auth_failures = { field = "authFailuresDelta", description = "Rejected or timed-out outer relay authentication stages." } + reconnects = { field = "reconnectsDelta", description = "Desktop control rebinds or generation replacements." } + sql_queries = { field = "sqlQueriesDelta", description = "Completed relay SQL operations." } + sql_failures = { field = "sqlFailuresDelta", description = "Failed relay SQL operations." } + db_pool_total = { field = "databasePoolTotal", description = "Open PostgreSQL connections in the process pool." } + db_pool_idle = { field = "databasePoolIdle", description = "Idle PostgreSQL connections in the process pool." } + db_pool_waiting = { field = "databasePoolWaiting", description = "Current requests queued for a PostgreSQL connection." } + db_waiters_max = { field = "databasePoolWaitersMax", description = "Maximum requests queued for a PostgreSQL connection during the interval." } + db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." } + db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." } + } + relay_custom_alerts = { + connection_headroom = { + pages_oncall = true + metric = "total_connections" + # Live values, set by hand on 2026-08-05. The cell arm is ~85% of the 440 usable + # connection units (600 hard cap - 100 rebind reserve - 60 unobserved bound); 800 + # exceeded the 600 cap outright and could never fire. + threshold_run = 550 + threshold_gce = 374 + duration = "120s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "A relay process is above its reviewed connection warning point; stop new assignment to the cell and follow the hot-cell runbook." + } + queue_pressure = { + pages_oncall = false + metric = "queued_bytes" + threshold_run = 50331648 + threshold_gce = 50331648 + duration = "120s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "Process-wide queued ciphertext exceeded 75% of the 64 MiB hard budget. Investigate slow receivers before admission starts rejecting." + } + auth_failures = { + pages_oncall = false + metric = "auth_failures" + threshold_run = 20 + threshold_gce = 20 + duration = "0s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "Outer authentication failures exceeded the per-process interval threshold. Check auth/JWKS health and abuse sources without logging bearer values." + } + reconnects = { + pages_oncall = false + metric = "reconnects" + threshold_run = 100 + threshold_gce = 100 + duration = "0s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "Relay control reconnects exceeded the per-process interval threshold. Check revision churn, GFE terminations, and reconnect jitter." + } + sql_failures = { + pages_oncall = true + # Tuned live on 2026-08-05 to stop Slack alert noise; codified here so an apply cannot revert it. + metric = "sql_failures" + threshold_run = 0 + threshold_gce = 0 + duration = "300s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "A relay SQL operation failed. Check Cloud SQL availability, connection pressure, and transaction retry outcomes." + } + sql_latency = { + pages_oncall = false + metric = "sql_latency_ms" + threshold_run = 500 + threshold_gce = 500 + duration = "120s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "Relay SQL operations remained above 500 ms. Inspect lock contention and Cloud SQL health before migrations or drains." + } + database_pool_waiters = { + pages_oncall = true + # Tuned live on 2026-08-05 to stop Slack alert noise; codified here so an apply cannot revert it. + metric = "db_waiters_max" + threshold_run = 5 + threshold_gce = 5 + duration = "300s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "A relay process queued work for a PostgreSQL connection. Check public-assignment admission, pool saturation, and Cloud SQL latency before scaling." + } + database_pool_wait = { + pages_oncall = true + # Tuned live on 2026-08-05 to stop Slack alert noise; codified here so an apply cannot revert it. + metric = "db_wait_ms_max" + threshold_run = 500 + threshold_gce = 500 + duration = "300s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "A relay process waited over 500 ms for a PostgreSQL connection. Check long transactions and lock contention before migrations or drains." + } + http_latency = { + pages_oncall = false + metric = "http_latency_ms" + threshold_run = 2000 + threshold_gce = 2000 + duration = "120s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "Relay HTTP handling remained above two seconds. Check director assignment/resolve latency and SQL contention; WebSocket lifetimes are intentionally excluded." + } + heap_pressure = { + pages_oncall = false + metric = "heap_used_bytes" + threshold_run = 419430400 + threshold_gce = 419430400 + duration = "120s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "Node.js heap stayed above 400 MiB on a 512 MiB container. Drain the affected cell and investigate connection or queue retention." + } + event_loop_delay = { + pages_oncall = false + metric = "event_loop_ms_p99" + threshold_run = 250 + threshold_gce = 250 + duration = "120s" + aligner = "ALIGN_PERCENTILE_99" + reducer = "REDUCE_MAX" + documentation = "Relay event-loop p99 delay stayed above 250 ms. Check CPU, SQL callbacks, synchronized heartbeats, and queue pressure." + } + } +} + +resource "google_logging_metric" "relay_snapshot" { + for_each = local.relay_runtime_metrics + + project = var.project_id + name = "orca_relay_${each.key}" + description = each.value.description + filter = local.relay_runtime_log_filter + value_extractor = "EXTRACT(jsonPayload.${each.value.field})" + label_extractors = { + role = "EXTRACT(jsonPayload.role)" + cell_id = "EXTRACT(jsonPayload.cellId)" + region = "EXTRACT(jsonPayload.region)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "DISTRIBUTION" + unit = contains(["sql_latency_ms", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1" + + labels { + key = "role" + value_type = "STRING" + description = "Relay process role." + } + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier." + } + + labels { + key = "region" + value_type = "STRING" + description = "Coarse Relay region." + } + } + + bucket_options { + exponential_buckets { + num_finite_buckets = 24 + growth_factor = 2 + scale = 1 + } + } +} + +resource "google_logging_metric" "relay_incident" { + for_each = local.relay_incident_metrics + + project = var.project_id + name = "orca_relay_${each.key}" + description = each.value.description + filter = each.value.filter + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + } +} + +resource "google_monitoring_alert_policy" "relay_custom" { + for_each = local.relay_custom_alerts + + project = var.project_id + display_name = "Orca Relay: ${replace(each.key, "_", " ")}" + combiner = "OR" + enabled = true + # Why: only the reviewed critical alerts page; the rest stay in the console. Applying one + # list to every policy would have put the noisy ones back into Slack. + notification_channels = each.value.pages_oncall ? var.relay_alert_notification_channels : [] + + conditions { + display_name = each.key + + condition_threshold { + filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_${each.value.metric}\"" + comparison = "COMPARISON_GT" + threshold_value = each.value.threshold_run + duration = each.value.duration + + aggregations { + alignment_period = "300s" + per_series_aligner = each.value.aligner + cross_series_reducer = each.value.reducer + group_by_fields = ["resource.label.\"service_name\""] + } + + trigger { + count = 1 + } + } + } + + dynamic "conditions" { + for_each = each.key == "connection_headroom" ? [] : [each.value] + content { + display_name = "${each.key} (GCE cell)" + + condition_threshold { + filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_${conditions.value.metric}\"" + comparison = "COMPARISON_GT" + threshold_value = conditions.value.threshold_gce + duration = conditions.value.duration + + aggregations { + alignment_period = "300s" + per_series_aligner = conditions.value.aligner + cross_series_reducer = conditions.value.reducer + group_by_fields = ["resource.label.\"instance_id\""] + } + + trigger { + count = 1 + } + } + } + } + + documentation { + content = each.value.documentation + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_snapshot] +} + +resource "google_monitoring_alert_policy" "relay_assignment_5xx" { + project = var.project_id + display_name = "Orca Relay: assignment 5xx" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "assignment 5xx" + + condition_threshold { + filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_assignment_5xx\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["resource.label.\"service_name\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A desktop could not obtain a Relay assignment. Check PostgreSQL retry/exhaustion signals and director SQL latency; do not restart GCE cells or invalidate pairings." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_incident] +} + +resource "google_monitoring_alert_policy" "relay_gce_connection_headroom" { + for_each = local.relay_gce_connection_warning_groups + + project = var.project_id + display_name = "Orca Relay: connection headroom (GCE ${each.key})" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "connection headroom (GCE ${each.key})" + + condition_threshold { + filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_total_connections\" AND (${join(" OR ", [for cell_id in each.value : "metric.label.\"cell_id\"=\"${cell_id}\""])})" + comparison = "COMPARISON_GT" + threshold_value = tonumber(each.key) + duration = "120s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_PERCENTILE_99" + cross_series_reducer = "REDUCE_MAX" + group_by_fields = ["resource.label.\"instance_id\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A Relay GCE cell is above 85% of its configured ordinary placement ceiling; stop new assignment to the cell and follow the hot-cell runbook." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_snapshot] +} + +resource "google_monitoring_alert_policy" "relay_assignment_edge_429" { + project = var.project_id + display_name = "Orca Relay: assignment edge 429" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "assignment edge 429" + + condition_threshold { + filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_assignment_edge_429\"" + comparison = "COMPARISON_GT" + threshold_value = 100 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["resource.label.\"service_name\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "Cloud Run rejected a sustained assignment burst before an instance was available. Check public-assignment admission, director concurrency, and database latency before scaling." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_incident] +} + +resource "google_monitoring_alert_policy" "relay_postgres_retry_exhausted" { + project = var.project_id + display_name = "Orca Relay: PostgreSQL retry exhausted" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "PostgreSQL retry exhausted (Cloud Run)" + + condition_threshold { + filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_postgres_retry_exhausted\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "180s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["resource.label.\"service_name\""] + } + + trigger { + count = 1 + } + } + } + + conditions { + display_name = "PostgreSQL retry exhausted (GCE cell)" + + condition_threshold { + filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_postgres_retry_exhausted\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "180s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["resource.label.\"instance_id\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A Relay database transaction remained unsuccessful after bounded whole-transaction retry. Check Cloud SQL deadlocks/locks and customer-visible request failures before changing cell admission." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_incident] +} + +resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" { + project = var.project_id + display_name = "Orca Relay: Cloud SQL connection headroom" + combiner = "OR" + enabled = true + + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "Cloud SQL backends above 320" + + condition_threshold { + filter = "resource.type=\"cloudsql_database\" AND resource.label.\"database_id\"=\"${var.project_id}:${local.relay_database_instance_name}\" AND metric.type=\"cloudsql.googleapis.com/database/postgresql/num_backends\"" + comparison = "COMPARISON_GT" + threshold_value = 320 + duration = "300s" + + aggregations { + alignment_period = "60s" + per_series_aligner = "ALIGN_MAX" + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "Cloud SQL connections exceeded 80% of the 400-connection ceiling. Pause Relay pool or cell growth and inspect pool waits before the modeled 385-connection operating maximum is reached." + mime_type = "text/markdown" + } +} diff --git a/cloud/infra/terraform/relay-shared.tf b/cloud/infra/terraform/relay-shared.tf new file mode 100644 index 00000000000..d4f0e5dac15 --- /dev/null +++ b/cloud/infra/terraform/relay-shared.tf @@ -0,0 +1,90 @@ +# Values the relay root shares with the foundation and apps roots, expressed as literals or +# data lookups so the relay never references another root's resources. Every literal here +# renders byte-identically to the resource attribute it replaces; the partition test pins that. +locals { + relay_shared_labels = { + app = "orca-cloud" + environment = var.environment + managed_by = "terraform" + } + relay_github_repository = "${var.github_owner}/${var.github_repo}" + relay_github_repository_claims = [ + "assertion.repository == '${local.relay_github_repository}'", + "assertion.repository_id == '${var.github_repo_id}'", + "assertion.repository_owner_id == '${var.github_owner_id}'", + ] + + # Dual accept during the public extraction: the primary repository first, then every repository + # var.github_accepted_repositories adds. Each one renders its own OR arm in every provider + # condition, so both repos can run the same workflows through the same identities. A repository + # that imports these workflows may rename the files, hence the per-repository prefix. + relay_github_accepted_repositories = concat([{ + owner = var.github_owner + repo = var.github_repo + repo_id = var.github_repo_id + owner_id = var.github_owner_id + workflow_file_prefix = "" + }], var.github_accepted_repositories) + + relay_github_single_repository = length(local.relay_github_accepted_repositories) == 1 + + relay_github_accepted_repository_names = [ + for repository in local.relay_github_accepted_repositories : + "${repository.owner}/${repository.repo}" + ] + + # Everything before the workflow file name, per accepted repository. + relay_github_workflow_ref_prefixes = [ + for repository in local.relay_github_accepted_repositories : + "${repository.owner}/${repository.repo}/.github/workflows/${repository.workflow_file_prefix}" + ] + + # The three repository claims as one conjunction, per accepted repository, for the OR arms. + relay_github_accepted_repository_claims = [ + for repository in local.relay_github_accepted_repositories : + join(" && ", [ + "assertion.repository == '${repository.owner}/${repository.repo}'", + "assertion.repository_id == '${repository.repo_id}'", + "assertion.repository_owner_id == '${repository.owner_id}'" + ]) + ] + + # With one accepted repository the claims lead each condition exactly as they always have. With + # more they move inside the arms, because a leading claim would contradict the other arm. + relay_github_leading_repository_claims = ( + local.relay_github_single_repository ? local.relay_github_repository_claims : [] + ) + + relay_create_github_deploy_identity = var.github_owner != "" && var.github_repo != "" + relay_create_production_ops_identity = local.relay_create_github_deploy_identity && var.environment == "production" + + # google_service_account.github_deploy lives in the relay root in production and in the apps + # root in staging, so its email is derived rather than read, matching the runtime accounts above. + # Staging Relay workflows authenticate as the relay-owned github_staging_relay_deploy account + # instead, so every relay binding, the cell startup metadata, and the director env follow it. + relay_github_deploy_service_account_email = ( + var.environment == "production" + ? "${var.name_prefix}-gha-deploy@${var.project_id}.iam.gserviceaccount.com" + : "${var.name_prefix}-gha-relay@${var.project_id}.iam.gserviceaccount.com" + ) + relay_github_deploy_service_account_member = "serviceAccount:${local.relay_github_deploy_service_account_email}" + + relay_workload_identity_pool_id = "${var.name_prefix}-github" + relay_workload_identity_pool_name = "projects/${data.google_project.relay.number}/locations/global/workloadIdentityPools/${local.relay_workload_identity_pool_id}" + + # Cloud SQL instance is foundation-owned; cell plans already derive the connection name so + # they stay independent of database drift. + relay_database_instance_name = "${var.name_prefix}-auth-db" + relay_database_connection_name = "${var.project_id}:${var.region}:${local.relay_database_instance_name}" +} + +data "google_project" "relay" { + project_id = var.project_id +} + +# Existence checks only: nothing in a plan value depends on these reads. +data "google_artifact_registry_repository" "relay_images" { + project = var.project_id + location = var.region + repository_id = var.artifact_repository_id +} diff --git a/cloud/infra/terraform/relay-staging-deploy-iam.tf b/cloud/infra/terraform/relay-staging-deploy-iam.tf new file mode 100644 index 00000000000..ac5f5632d57 --- /dev/null +++ b/cloud/infra/terraform/relay-staging-deploy-iam.tf @@ -0,0 +1,170 @@ +# The staging Relay deploy identity. +# +# In production the shared `github_deploy` account is relay-owned; in staging it belongs to +# infra/terraform-apps and four app workflows can also mint it. These resources give the five +# staging Relay workflows their own account, provider, and bindings so the relay root owns every +# credential its own workflows use. +# +# Bindings that already name local.relay_github_deploy_service_account_member follow that local +# (relay-shared.tf) and are NOT repeated here: the staging power custom role, serviceAccountUser +# on relay_runtime and relay_director_runtime, and the three regional-placement secret bindings. +# Everything below replaces a grant the apps root still makes to `github_deploy`. + +locals { + create_staging_relay_deploy_identity = ( + local.relay_create_github_deploy_identity && var.environment == "staging" + ) + github_staging_relay_deploy_workflow_files = [ + "bootstrap-relay-staging-capacity.yml", + "deploy-relay-staging-gce-candidate.yml", + "deploy-relay-staging.yml", + "operate-relay-asia-admission.yml", + "power-relay-staging.yml" + ] + github_staging_relay_deploy_workflow_clauses = [ + for prefix in local.relay_github_workflow_ref_prefixes : + "(${join(" || ", [for workflow_file in local.github_staging_relay_deploy_workflow_files : "assertion.workflow_ref == '${prefix}${workflow_file}@refs/heads/main'"])})" + ] + # Apps-owned account the shared staging power workflow scales to zero alongside the director. + staging_auth_runtime_service_account_email = "${var.name_prefix}-auth@${var.project_id}.iam.gserviceaccount.com" +} + +resource "google_service_account" "github_staging_relay_deploy" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + project = var.project_id + account_id = "${var.name_prefix}-gha-relay" + display_name = "Orca Relay staging deploy" + description = "Runs the exact reviewed Relay staging deploy, candidate, power, and admission workflows." +} + +resource "google_iam_workload_identity_pool_provider" "github_staging_relay_deploy" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + project = var.project_id + workload_identity_pool_id = local.relay_workload_identity_pool_id + workload_identity_pool_provider_id = "github-relay-deploy" + display_name = "GitHub Relay staging deploy" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.repository_id" = "assertion.repository_id" + "attribute.repository_owner_id" = "assertion.repository_owner_id" + "attribute.ref" = "assertion.ref" + "attribute.environment" = "assertion.environment" + "attribute.workflow_ref" = "assertion.workflow_ref" + "attribute.relay_ops_identity" = "'staging-deploy'" + } + + # An allowlist of exact workflow refs, never a prefix: a namespace grant would hand this account + # to any future staging workflow with no Terraform diff. + attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ + "assertion.ref == 'refs/heads/main'", + "assertion.environment == 'staging'", + local.relay_github_workflow_conditions["github_staging_relay_deploy"] + ])) + + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account_iam_member" "github_staging_relay_deploy_workload_identity_user" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + service_account_id = google_service_account.github_staging_relay_deploy[0].name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/staging-deploy" +} + +# Every one of the five runs `terraform init` (or `infra.mjs init --env staging`) first, and the +# GCS backend lists the bucket before it can open the state. Bucket metadata only, no object +# access; this mirrors relay_fence_broker_bucket_reader. +resource "google_storage_bucket_iam_member" "github_staging_relay_deploy_state_list" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + bucket = "${var.project_id}-terraform-state" + role = "roles/storage.legacyBucketReader" + member = google_service_account.github_staging_relay_deploy[0].member +} + +# Reads reviewed topology: `terraform output -json relay_gce_cell_deployments` and the +# `terraform console` binds in Deploy Relay Staging. No step in the five applies, so this is +# objectViewer, not the capacity identity's objectAdmin, over the same two objects. +resource "google_storage_bucket_iam_member" "github_staging_relay_deploy_state" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + bucket = "${var.project_id}-terraform-state" + role = "roles/storage.objectViewer" + member = google_service_account.github_staging_relay_deploy[0].member + + condition { + title = "relay_staging_deploy_state" + description = "Limits the staging Relay deploy workflows to the default Terraform state and lock." + expression = join(" || ", [ + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", + "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" + ]) + } +} + +# Deploy Relay Staging step "Require the mirrored immutable image" runs +# `gcloud artifacts docker images describe`; nothing in the five writes to the repository. +resource "google_artifact_registry_repository_iam_member" "github_staging_relay_deploy_artifact_reader" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + project = var.project_id + location = var.region + repository = var.artifact_repository_id + role = "roles/artifactregistry.reader" + member = google_service_account.github_staging_relay_deploy[0].member +} + +# Deploy Relay Staging step "Deploy director blue/green", Operate Relay Asia Admission step +# "Deploy the registered additive director topology", and Power Relay Staging's scale-to-zero all +# drive `gcloud run services update|update-traffic` against the staging director. +resource "google_cloud_run_v2_service_iam_member" "github_staging_relay_deploy_director_developer" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + project = var.project_id + location = var.region + name = var.relay_cloud_run_service_name + role = "roles/run.developer" + member = google_service_account.github_staging_relay_deploy[0].member +} + +# Power Relay Staging step "Inspect or change staging power state" scales both entries of +# CLOUD_RUN_SERVICES in power-staging-relay.mjs to zero, and the second one is the shared staging +# auth service. The same workflow already stops the shared staging database through +# orcaRelayStagingPower, so this stays with the power operator rather than the apps root. +resource "google_cloud_run_v2_service_iam_member" "github_staging_relay_deploy_auth_developer" { + count = local.create_staging_relay_deploy_identity && var.relay_staging_power_auth_service_name != "" ? 1 : 0 + + project = var.project_id + location = var.region + name = var.relay_staging_power_auth_service_name + role = "roles/run.developer" + member = google_service_account.github_staging_relay_deploy[0].member +} + +# That same scale-to-zero mints a revision when the latest ready one is not already at zero, and +# Cloud Run gates a revision on actAs over the service's runtime account. +resource "google_service_account_iam_member" "github_staging_relay_deploy_auth_runtime_user" { + count = local.create_staging_relay_deploy_identity && var.relay_staging_power_auth_service_name != "" ? 1 : 0 + + service_account_id = "projects/${var.project_id}/serviceAccounts/${local.staging_auth_runtime_service_account_email}" + role = "roles/iam.serviceAccountUser" + member = google_service_account.github_staging_relay_deploy[0].member +} + +# Deploy Relay Staging GCE Candidate preflight reads MIG, instance, and backend-service topology +# (deploy-relay-gce-candidate.mjs inspectCell), which the power role's instanceGroupManagers.get +# does not cover. +resource "google_project_iam_member" "github_staging_relay_deploy_compute_viewer" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + project = var.project_id + role = "roles/compute.viewer" + member = google_service_account.github_staging_relay_deploy[0].member +} diff --git a/cloud/infra/terraform/relay.tf b/cloud/infra/terraform/relay.tf new file mode 100644 index 00000000000..7a5124a00b1 --- /dev/null +++ b/cloud/infra/terraform/relay.tf @@ -0,0 +1,577 @@ +resource "google_service_account" "relay_runtime" { + project = var.project_id + account_id = "${var.name_prefix}-relay" + display_name = var.environment == "staging" ? "Orca Relay" : "Orca Relay cells" + description = var.environment == "staging" ? ( + "Runtime identity for the Orca Relay director and stamped cells." + ) : "Runtime identity for stamped Orca Relay cells." +} + +resource "google_service_account" "relay_director_runtime" { + project = var.project_id + account_id = "${var.name_prefix}-relay-dir" + display_name = "Orca Relay director" + description = "Runtime and regional rehoming caller identity for the Orca Relay director." +} + +resource "google_project_iam_member" "relay_runtime_cloudsql_client" { + project = var.project_id + role = "roles/cloudsql.client" + member = google_service_account.relay_runtime.member +} + +resource "google_project_iam_member" "relay_director_runtime_cloudsql_client" { + project = var.project_id + role = "roles/cloudsql.client" + member = google_service_account.relay_director_runtime.member +} + +resource "random_password" "relay_assignment_signing_key" { + length = 48 + special = false +} + +resource "google_secret_manager_secret" "relay_assignment_signing_key" { + project = var.project_id + secret_id = "orca-cloud-relay-assignment-signing-key" + labels = local.relay_shared_labels + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "relay_assignment_signing_key" { + secret = google_secret_manager_secret.relay_assignment_signing_key.id + secret_data = random_password.relay_assignment_signing_key.result +} + +resource "google_secret_manager_secret_iam_member" "relay_assignment_signing_key_accessor" { + project = var.project_id + secret_id = google_secret_manager_secret.relay_assignment_signing_key.secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_runtime.member +} + +resource "google_secret_manager_secret_iam_member" "relay_assignment_signing_key_director_accessor" { + project = var.project_id + secret_id = google_secret_manager_secret.relay_assignment_signing_key.secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_director_runtime.member +} + +resource "google_secret_manager_secret" "relay_regional_placement_enabled" { + project = var.project_id + secret_id = "orca-cloud-relay-regional-placement-enabled" + labels = local.relay_shared_labels + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "relay_regional_placement_enabled" { + secret = google_secret_manager_secret.relay_regional_placement_enabled.id + secret_data = tostring(var.relay_regional_placement_enabled) +} + +resource "google_secret_manager_secret_iam_member" "relay_regional_placement_runtime_accessor" { + project = var.project_id + secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_runtime.member +} + +resource "google_secret_manager_secret_iam_member" "relay_regional_placement_director_accessor" { + project = var.project_id + secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_director_runtime.member +} + +resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_accessor" { + count = local.relay_create_github_deploy_identity ? 1 : 0 + + project = var.project_id + secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id + role = "roles/secretmanager.secretAccessor" + member = local.relay_github_deploy_service_account_member +} + +resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_adder" { + count = local.relay_create_github_deploy_identity ? 1 : 0 + + project = var.project_id + secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id + role = "roles/secretmanager.secretVersionAdder" + member = local.relay_github_deploy_service_account_member +} + +resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_viewer" { + count = local.relay_create_github_deploy_identity ? 1 : 0 + + project = var.project_id + secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id + role = "roles/secretmanager.viewer" + member = local.relay_github_deploy_service_account_member +} + +data "external" "relay_serving_regional_placement_version" { + program = [ + "node", + "${path.module}/../../dev/scripts/read-relay-serving-regional-placement-version.mjs" + ] + query = { + project = var.project_id + region = var.region + service = var.relay_cloud_run_service_name + bootstrap_version = google_secret_manager_secret_version.relay_regional_placement_enabled.version + } +} + +resource "google_cloud_run_v2_service" "relay" { + project = var.project_id + name = var.relay_cloud_run_service_name + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + invoker_iam_disabled = true + labels = local.relay_shared_labels + + template { + service_account = google_service_account.relay_director_runtime.email + timeout = "${var.relay_director_request_timeout_seconds}s" + max_instance_request_concurrency = var.relay_director_concurrency + + scaling { + min_instance_count = var.relay_min_instances + max_instance_count = var.relay_max_instances + } + + volumes { + name = "cloudsql" + + cloud_sql_instance { + instances = [local.relay_database_connection_name] + } + } + + containers { + image = var.relay_cloud_run_image + + env { + name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.relay_regional_placement_enabled.secret_id + version = data.external.relay_serving_regional_placement_version.result.version + } + } + } + + ports { + container_port = 8080 + } + + volume_mounts { + name = "cloudsql" + mount_path = "/cloudsql" + } + + env { + name = "DATABASE_URL" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.relay_database_url.secret_id + version = "latest" + } + } + } + + env { + name = "ORCA_RELAY_ASSIGNMENT_SIGNING_KEY" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.relay_assignment_signing_key.secret_id + version = "latest" + } + } + } + + env { + name = "ORCA_RELAY_PUBLIC_URL" + value = var.relay_base_url + } + + env { + name = "ORCA_RELAY_CELL_URL" + value = var.relay_base_url + } + + env { + name = "ORCA_RELAY_AUTH_ISSUER" + value = var.auth_base_url + } + + env { + name = "ORCA_RELAY_AUTH_AUDIENCE" + value = "orca-relay" + } + + env { + name = "ORCA_RELAY_JWKS_URL" + value = "${var.auth_base_url}/.well-known/jwks.json" + } + + env { + name = "ORCA_RELAY_ROLE" + value = "director" + } + + env { + name = "ORCA_RELAY_ADMISSION_SELECTOR_VERSION" + value = "3" + } + + env { + name = "ORCA_RELAY_CELL_ID" + value = "director" + } + + env { + name = "ORCA_RELAY_CELLS_JSON" + value = local.relay_director_cells_json + } + + env { + name = "ORCA_RELAY_ADMIN_AUDIENCE" + value = "${var.relay_base_url}/v1/admin/drain" + } + + env { + name = "ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT" + value = local.relay_github_deploy_service_account_email + } + + env { + name = "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" + value = local.relay_capacity_service_account_email + } + + env { + name = "ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT" + value = local.relay_asia_proof_service_account_email + } + + env { + name = "ORCA_RELAY_MONITOR_SERVICE_ACCOUNT" + value = try(google_service_account.github_monitor[0].email, "") + } + + env { + name = "ORCA_RELAY_FENCE_SERVICE_ACCOUNT" + value = try(google_service_account.github_fence[0].email, "") + } + + env { + name = "ORCA_RELAY_FENCE_BROKER_SERVICE_ACCOUNT" + value = try(google_service_account.relay_fence_broker[0].email, "") + } + + env { + name = "ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT" + value = google_service_account.relay_runtime.email + } + + env { + name = "ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT" + value = google_service_account.relay_director_runtime.email + } + + env { + name = "ORCA_RELAY_REHOME_AUDIENCE" + value = "${var.relay_base_url}/v1/admin/host-drain" + } + + env { + name = "ORCA_RELAY_HEARTBEAT_AUDIENCE" + value = "${var.relay_base_url}/v1/admin/cell-heartbeat" + } + + env { + name = "ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED" + value = tostring(var.relay_public_assignments_enabled) + } + + env { + name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY" + value = tostring(var.relay_public_assignment_concurrency) + } + + env { + name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS" + value = tostring(var.relay_public_assignment_retry_after_seconds) + } + + env { + name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX" + value = tostring(var.relay_public_assignment_queue_max) + } + + env { + name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS" + value = tostring(var.relay_public_assignment_wait_ms) + } + + env { + name = "ORCA_RELAY_DATABASE_POOL_MAX" + value = tostring(var.relay_director_database_pool_max) + } + + env { + name = "ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY" + value = tostring(var.relay_public_sticky_concurrency) + } + + env { + name = "ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX" + value = tostring(var.relay_public_sticky_queue_max) + } + + env { + name = "ORCA_RELAY_PUBLIC_STICKY_WAIT_MS" + value = tostring(var.relay_public_sticky_wait_ms) + } + + env { + name = "ORCA_RELAY_PUBLIC_STICKY_RETRY_AFTER_SECONDS" + value = tostring(var.relay_public_sticky_retry_after_seconds) + } + + resources { + limits = { + cpu = var.relay_cloud_run_cpu + memory = var.relay_cloud_run_memory + } + + cpu_idle = false + } + } + } + + # Deploys update immutable images; Terraform owns service shape and IAM. + lifecycle { + # Why: the relay refuses to boot unless both admission lanes fit the pool, so catch it + # at plan time rather than as a crash loop on the candidate revision. + precondition { + condition = (var.relay_public_assignment_concurrency + + var.relay_public_sticky_concurrency) <= var.relay_director_database_pool_max + error_message = "Placement plus reconnect admission must fit the director database pool." + } + + ignore_changes = [ + client, + client_version, + template[0].containers[0].image + ] + } + + depends_on = [ + data.google_artifact_registry_repository.relay_images, + google_project_iam_member.relay_director_runtime_cloudsql_client, + google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor, + google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor, + google_secret_manager_secret_iam_member.relay_database_url_director_accessor, + google_secret_manager_secret_version.relay_assignment_signing_key, + google_secret_manager_secret_version.relay_regional_placement_enabled, + google_secret_manager_secret_version.relay_database_url + ] +} + +resource "google_cloud_run_v2_service" "relay_cell" { + for_each = var.relay_cells + + project = var.project_id + name = each.value.service_name + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + invoker_iam_disabled = true + # Require an explicit configuration change before a stamped cell can be decommissioned. + deletion_protection = each.value.deletion_protection + labels = merge(local.relay_shared_labels, { + "orca-relay-role" = "cell" + "orca-relay-cell" = each.key + }) + + template { + service_account = google_service_account.relay_runtime.email + timeout = "${var.relay_request_timeout_seconds}s" + max_instance_request_concurrency = var.relay_concurrency + + scaling { + min_instance_count = each.value.min_instances + max_instance_count = each.value.max_instances + } + + volumes { + name = "cloudsql" + + cloud_sql_instance { + instances = [local.relay_database_connection_name] + } + } + + containers { + image = var.relay_cloud_run_image + + ports { + container_port = 8080 + } + + volume_mounts { + name = "cloudsql" + mount_path = "/cloudsql" + } + + env { + name = "DATABASE_URL" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.relay_database_url.secret_id + version = "latest" + } + } + } + + env { + name = "ORCA_RELAY_ASSIGNMENT_SIGNING_KEY" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.relay_assignment_signing_key.secret_id + version = "latest" + } + } + } + + env { + name = "ORCA_RELAY_PUBLIC_URL" + value = each.value.url + } + + env { + name = "ORCA_RELAY_CELL_URL" + value = each.value.url + } + + env { + name = "ORCA_RELAY_AUTH_ISSUER" + value = var.auth_base_url + } + + env { + name = "ORCA_RELAY_AUTH_AUDIENCE" + value = "orca-relay" + } + + env { + name = "ORCA_RELAY_JWKS_URL" + value = "${var.auth_base_url}/.well-known/jwks.json" + } + + env { + name = "ORCA_RELAY_ROLE" + value = "cell" + } + + env { + name = "ORCA_RELAY_CELL_ID" + value = each.key + } + + env { + name = "ORCA_RELAY_CELL_CAPACITY" + value = tostring(each.value.capacity_requests) + } + + env { + name = "ORCA_RELAY_CELLS_JSON" + value = "[]" + } + + env { + name = "ORCA_RELAY_ADMIN_AUDIENCE" + value = "${var.relay_base_url}/v1/admin/drain" + } + + env { + name = "ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT" + value = local.relay_github_deploy_service_account_email + } + + env { + name = "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" + value = local.relay_capacity_service_account_email + } + + env { + name = "ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT" + value = local.relay_asia_proof_service_account_email + } + + env { + name = "ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT" + value = google_service_account.relay_runtime.email + } + + env { + name = "ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT" + value = contains(var.relay_region_rehome_source_cell_ids, each.key) ? google_service_account.relay_director_runtime.email : "" + } + + env { + name = "ORCA_RELAY_REHOME_AUDIENCE" + value = contains(var.relay_region_rehome_source_cell_ids, each.key) ? "${var.relay_base_url}/v1/admin/host-drain" : "" + } + + env { + name = "ORCA_RELAY_DIRECTOR_URL" + value = var.relay_base_url + } + + env { + name = "ORCA_RELAY_HEARTBEAT_AUDIENCE" + value = "${var.relay_base_url}/v1/admin/cell-heartbeat" + } + + resources { + limits = { + cpu = var.relay_cloud_run_cpu + memory = var.relay_cloud_run_memory + } + + cpu_idle = false + } + } + } + + lifecycle { + ignore_changes = [ + client, + client_version, + template[0].containers[0].image + ] + } + + depends_on = [ + data.google_artifact_registry_repository.relay_images, + google_project_iam_member.relay_runtime_cloudsql_client, + google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor, + google_secret_manager_secret_iam_member.relay_database_url_accessor, + google_secret_manager_secret_version.relay_assignment_signing_key, + google_secret_manager_secret_version.relay_database_url + ] +} diff --git a/cloud/infra/terraform/variables.tf b/cloud/infra/terraform/variables.tf new file mode 100644 index 00000000000..c3e06ee280f --- /dev/null +++ b/cloud/infra/terraform/variables.tf @@ -0,0 +1,468 @@ +variable "artifact_repository_id" { + type = string + description = "Artifact Registry Docker repository ID." +} + +variable "environment" { + type = string + description = "Deployment environment." + + validation { + condition = contains(["staging", "production"], var.environment) + error_message = "environment must be staging or production." + } +} + +variable "github_owner" { + type = string + description = "GitHub owner allowed to deploy through Workload Identity Federation." + default = "stablyai" +} + +variable "github_repo" { + type = string + description = "GitHub repo allowed to deploy through Workload Identity Federation." + default = "orca-cloud" +} + +# Numeric IDs survive a rename or transfer of the repository; every provider pins them next to the name. +variable "github_repo_id" { + type = string + description = "Numeric GitHub repository ID of github_owner/github_repo." + default = "1273841466" + + validation { + condition = can(regex("^[0-9]+$", var.github_repo_id)) + error_message = "github_repo_id must be the numeric repository ID." + } +} + +variable "github_owner_id" { + type = string + description = "Numeric GitHub owner ID of github_owner." + default = "127256420" + + validation { + condition = can(regex("^[0-9]+$", var.github_owner_id)) + error_message = "github_owner_id must be the numeric owner ID." + } +} + +# Additional repositories whose identical workflows the same identities must accept while the +# public extraction runs. Each entry renders its own OR arm in every provider condition, so the +# private repo keeps working while the public one takes over. `workflow_file_prefix` is the rename +# the importing repository applies to the workflow files it copies. Empty is the steady state: +# the final step of the cutover is to empty this list again and point github_owner/github_repo, +# github_repo_id, and github_owner_id at the surviving repository. +variable "github_accepted_repositories" { + type = list(object({ + owner = string + repo = string + repo_id = string + owner_id = string + workflow_file_prefix = string + })) + description = "Extra repositories accepted alongside github_owner/github_repo during the public extraction." + default = [] + + validation { + condition = alltrue([ + for repository in var.github_accepted_repositories : + can(regex("^[0-9]+$", repository.repo_id)) && can(regex("^[0-9]+$", repository.owner_id)) + ]) + error_message = "github_accepted_repositories entries must carry numeric repo_id and owner_id values." + } + + validation { + condition = alltrue([ + for repository in var.github_accepted_repositories : + can(regex("^[a-z0-9-]*$", repository.workflow_file_prefix)) + ]) + error_message = "github_accepted_repositories workflow_file_prefix must be lowercase letters, digits, or hyphens." + } +} + +variable "name_prefix" { + type = string + description = "Prefix used for named resources." +} + +variable "project_id" { + type = string + description = "GCP project ID." +} + +variable "region" { + type = string + description = "GCP region for regional resources." + default = "us-central1" +} + +variable "auth_base_url" { + type = string + description = "Public base URL of the auth service; OAuth callbacks and JWT issuer derive from it." +} + +variable "manage_relay_domain_mapping" { + type = bool + description = "Manage the Google Cloud Run mapping independently of the Cloudflare record." + default = false +} + +variable "relay_base_url" { + type = string + description = "Public TLS origin of the stable relay director." +} + +variable "relay_cloud_run_service_name" { + type = string + description = "Cloud Run service name for Orca Relay." +} + +variable "relay_staging_power_auth_service_name" { + type = string + description = "Shared staging auth Cloud Run service that Power Relay Staging scales to zero; empty outside staging." + default = "" +} + +variable "relay_cloud_run_image" { + type = string + description = "Initial image for the Terraform-created relay Cloud Run service." + default = "us-docker.pkg.dev/cloudrun/container/hello" +} + +variable "relay_cloud_run_cpu" { + type = string + description = "CPU limit for the relay container." + default = "1" +} + +variable "relay_cloud_run_memory" { + type = string + description = "Memory limit for the relay container." + default = "512Mi" +} + +variable "relay_fence_broker_service_name" { + type = string + description = "Private Cloud Run service that owns reviewed Relay Terraform fences." + default = "orca-cloud-relay-fence" +} + +variable "relay_fence_broker_image" { + type = string + description = "Immutable image for the private Relay fence broker." + default = "us-docker.pkg.dev/cloudrun/container/hello" + + validation { + condition = ( + var.relay_fence_broker_image == "us-docker.pkg.dev/cloudrun/container/hello" || + can(regex("^[a-z0-9.-]+/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$", var.relay_fence_broker_image)) + ) + error_message = "relay_fence_broker_image must be the bootstrap image or an immutable digest." + } +} + +variable "relay_fence_source_cell_id" { + type = string + description = "Exact incident source cell accepted by the private fence broker." + default = "production-gce-c3" +} + +variable "relay_fence_failed_target_cell_id" { + type = string + description = "Exact failed registered target accepted by the private fence broker." + default = "production-gce-c12" +} + +variable "relay_fence_replacement_target_cell_id" { + type = string + description = "Exact replacement target accepted by the private fence broker." + default = "production-gce-c13" +} + +variable "relay_fence_unobserved_connection_bound" { + type = number + description = "Reviewed unobserved connection bound enforced during supersession." + default = 60 + + validation { + condition = ( + var.relay_fence_unobserved_connection_bound >= 0 && + var.relay_fence_unobserved_connection_bound < 500 + ) + error_message = "relay_fence_unobserved_connection_bound must be between zero and 499." + } +} + +variable "relay_director_concurrency" { + type = number + description = "Cloud Run concurrency for short-lived director HTTP requests." + default = 80 +} + +variable "relay_director_request_timeout_seconds" { + type = number + description = "Cloud Run timeout for short-lived director HTTP requests." + default = 30 +} + +variable "relay_concurrency" { + type = number + description = "Cloud Run cell concurrency; every WebSocket leg counts." + default = 1000 +} + +variable "relay_request_timeout_seconds" { + type = number + description = "Cloud Run cell request timeout for standing WebSocket legs." + default = 3600 +} + +variable "relay_public_assignments_enabled" { + type = bool + description = "Emergency switch for public assignment and resolve requests." + default = true +} + +variable "relay_regional_placement_enabled" { + type = bool + description = "Initial preferred-region placement state; audited director deploys own later changes." + default = true +} + +variable "relay_region_rehome_source_cell_ids" { + type = set(string) + description = "Reviewed US Relay cells allowed to advertise and accept the regional rehome source protocol." + default = [] +} + +variable "relay_public_assignment_concurrency" { + type = number + description = "Per-director public assignment operations allowed to reach shared state." + default = 2 +} + +variable "relay_public_assignment_retry_after_seconds" { + type = number + description = "Minimum retry interval enforced per relay host during assignment recovery." + default = 5 +} + +# Why: these three match the application defaults today. Pinning them keeps a code-side +# default change from silently re-tuning production on the next unrelated apply. +variable "relay_public_assignment_queue_max" { + type = number + description = "Queued public assignment operations allowed per director instance." + default = 128 +} + +variable "relay_public_assignment_wait_ms" { + type = number + description = "Milliseconds a public assignment waits for an admission slot before 503." + default = 4000 +} + +# Why: the sticky (reconnect) lane shared the assignment pool but lived only as a code +# default, so Terraform could not see it. Raising placement concurrency alone then pushed +# placement + sticky past the pool and the director refused to boot. +variable "relay_public_sticky_concurrency" { + type = number + description = "Per-director reconnect-lane operations allowed to reach shared state." + default = 1 +} + +variable "relay_public_sticky_queue_max" { + type = number + description = "Queued reconnect-lane operations allowed per director instance." + default = 64 +} + +variable "relay_public_sticky_wait_ms" { + type = number + description = "Milliseconds a reconnect waits for an admission slot before 503." + default = 2000 +} + +variable "relay_public_sticky_retry_after_seconds" { + type = number + description = "Minimum retry interval enforced per relay host during reconnect recovery." + default = 2 +} + +variable "relay_director_database_pool_max" { + type = number + description = "Director database pool size; must fit placement plus sticky admission slots." + default = 3 + + validation { + condition = var.relay_director_database_pool_max >= 3 + error_message = "The director pool must fit both placement and sticky admission slots." + } +} + +variable "relay_min_instances" { + type = number + description = "Minimum instances for the stable relay director." + default = 1 +} + +variable "relay_max_instances" { + type = number + description = "Maximum instances for the stateless stable relay director." + default = 2 + + validation { + condition = var.relay_max_instances >= 1 + error_message = "The relay director needs at least one instance." + } +} + +variable "relay_cells" { + type = map(object({ + service_name = string + url = string + capacity_requests = number + min_instances = number + max_instances = number + deletion_protection = optional(bool, true) + })) + description = "Explicit stamped max-one relay cells keyed by durable cell ID." + default = {} + + validation { + condition = alltrue([ + for cell in values(var.relay_cells) : + cell.max_instances == 1 && + cell.min_instances >= 0 && + cell.min_instances <= cell.max_instances && + cell.capacity_requests >= 1 && + cell.capacity_requests <= 1000 && + can(regex("^https://[^/]+$", cell.url)) + ]) + error_message = "Relay cells must use HTTPS origins, capacity 1..1000, and max exactly one." + } +} + +variable "relay_alert_notification_channels" { + type = list(string) + description = "Cloud Monitoring notification-channel resource names for Orca Relay alerts. Empty keeps policies visible without paging." + default = [] +} + +variable "relay_gce_domain" { + type = string + description = "Parent DNS name for GCE relay cells; each cell is one exact host below it." + default = "" + + validation { + condition = var.relay_gce_domain == "" || ( + can(regex("^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$", var.relay_gce_domain)) && + !startswith(var.relay_gce_domain, "*.") + ) + error_message = "relay_gce_domain must be empty or a lowercase DNS name without a wildcard or scheme." + } +} + +variable "relay_gce_subnetwork_cidr" { + type = string + description = "Private IPv4 range dedicated to GCE relay cells." + default = "10.42.0.0/24" + + validation { + condition = can(cidrhost(var.relay_gce_subnetwork_cidr, 1)) + error_message = "relay_gce_subnetwork_cidr must be a valid IPv4 CIDR." + } +} + +variable "relay_gce_additional_region_subnetwork_cidrs" { + type = map(string) + description = "Private IPv4 ranges for additive Relay regions; the primary region keeps its legacy resources." + default = {} + + validation { + condition = alltrue([ + for region, cidr in var.relay_gce_additional_region_subnetwork_cidrs : + contains(["asia-east2"], region) && + can(cidrhost(cidr, 1)) + ]) + error_message = "Additional Relay regions must be allowlisted and use valid IPv4 CIDRs." + } +} + +variable "relay_gce_cells" { + type = map(object({ + hostname = string + region = optional(string, "us-central1") + zone = string + machine_type = string + boot_disk_gb = number + boot_image = string + capacity_requests = number + database_pool_max = optional(number, 10) + image = string + initially_enabled = optional(bool, true) + connection_hard_cap = optional(number) + connection_unobserved_bound = optional(number) + })) + description = "Private GCE relay cells keyed by durable cell ID; unfenced cells remain fixed-one." + default = {} + + validation { + condition = alltrue([ + for cell_id, cell in var.relay_gce_cells : + can(regex("^[a-z][a-z0-9-]{0,39}$", cell_id)) && + can(regex("^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$", cell.hostname)) && + contains(["us-central1", "asia-east2"], cell.region) && + startswith(cell.zone, "${cell.region}-") && + can(regex("^[a-z0-9-]+$", cell.machine_type)) && + can(regex("^https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-[a-z0-9-]+$", cell.boot_image)) && + cell.boot_disk_gb >= 20 && + cell.boot_disk_gb <= 100 && + cell.capacity_requests >= 1 && + cell.capacity_requests <= 100000 && + cell.database_pool_max >= 1 && + cell.database_pool_max <= 100 && + ( + (cell.connection_hard_cap == null && + cell.connection_unobserved_bound == null) || + try( + contains([600, 1000, 3000], cell.connection_hard_cap) && + cell.connection_unobserved_bound >= 0 && + cell.connection_unobserved_bound < cell.connection_hard_cap - 100, + false + ) + ) && + can(regex("^[a-z0-9.-]+/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$", cell.image)) + ]) && length(distinct([for cell in values(var.relay_gce_cells) : cell.hostname])) == length(var.relay_gce_cells) + error_message = "GCE cells need an allowlisted region and matching zone, unique DNS labels, a pinned COS boot image, bounded machine/disk/capacity/pool values, paired supported connection limits with rebind headroom, and digest-pinned relay images." + } +} + +variable "relay_gce_cell_log_sample_rate" { + type = number + description = "Fraction of relay cell load-balancer requests written to Cloud Logging; 1 keeps assign-to-connection joins exact." + default = 1 + + validation { + condition = var.relay_gce_cell_log_sample_rate >= 0 && var.relay_gce_cell_log_sample_rate <= 1 + error_message = "relay_gce_cell_log_sample_rate must be between 0 and 1." + } +} + +variable "relay_gce_fenced_cells" { + type = set(string) + description = "Reviewed relay GCE cell IDs whose Terraform-owned MIG target size is zero." + default = [] +} + +variable "relay_gce_cloud_sql_proxy_image" { + type = string + description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers." + default = "gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:fc224915ef435afeb5b2a9421260a0d31986d5c8b7c7f5783c7f5d5885700cd2" + + validation { + condition = can(regex("^[a-z0-9.-]+/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$", var.relay_gce_cloud_sql_proxy_image)) + error_message = "relay_gce_cloud_sql_proxy_image must be pinned by sha256 digest." + } +} diff --git a/cloud/infra/terraform/versions.tf b/cloud/infra/terraform/versions.tf new file mode 100644 index 00000000000..730f785a87b --- /dev/null +++ b/cloud/infra/terraform/versions.tf @@ -0,0 +1,27 @@ +terraform { + required_version = ">= 1.7.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 6.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + + external = { + source = "hashicorp/external" + version = "~> 2.3" + } + } + + backend "gcs" {} +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/cloud/package.json b/cloud/package.json new file mode 100644 index 00000000000..c75d027d3d2 --- /dev/null +++ b/cloud/package.json @@ -0,0 +1,33 @@ +{ + "name": "orca-cloud", + "private": true, + "version": "0.0.0", + "packageManager": "pnpm@10.24.0", + "engines": { + "node": ">=24 <27", + "pnpm": ">=10" + }, + "scripts": { + "build": "pnpm -r build", + "dev": "pnpm --filter @orca-cloud/relay dev", + "incident:relay": "pnpm --filter @orca-cloud/relay-ops incident:monitor", + "incident:relay-preflight": "pnpm --filter @orca-cloud/relay-ops incident:preflight", + "infra:apply": "node dev/scripts/infra.mjs apply", + "infra:init": "node dev/scripts/infra.mjs init", + "infra:plan": "node dev/scripts/infra.mjs plan", + "lint": "pnpm -r lint", + "load:relay:controls": "node dev/scripts/load-relay-controls.mjs", + "load:relay:model": "node dev/scripts/run-relay-load-model.mjs", + "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", + "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", + "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", + "test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "typecheck": "pnpm -r typecheck" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "tsx": "^4.21.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/packages/relay-contract/package.json b/cloud/packages/relay-contract/package.json new file mode 100644 index 00000000000..4c224b51085 --- /dev/null +++ b/cloud/packages/relay-contract/package.json @@ -0,0 +1,23 @@ +{ + "name": "@orca-cloud/relay-contract", + "private": true, + "version": "0.0.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "pnpm clean && tsc -p tsconfig.build.json", + "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", + "lint": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/node": "^24.10.0", + "typescript": "^5.9.3", + "vitest": "^4.0.8" + } +} diff --git a/cloud/packages/relay-contract/src/admission-budgets.ts b/cloud/packages/relay-contract/src/admission-budgets.ts new file mode 100644 index 00000000000..6dcf18ec3af --- /dev/null +++ b/cloud/packages/relay-contract/src/admission-budgets.ts @@ -0,0 +1,76 @@ +export const RELAY_ADMISSION_BUDGETS = { + cloudRunConcurrency: 900, + maxPreAuthConnections: 45, + maxPreAuthPerSource: 4, + maxPreAuthAttemptsPerSourcePerMinute: 30, + reservedHostControls: 100, + reservedHostDataSockets: 150, + maxProcessQueuedBytes: 64 * 1024 * 1024, + spliceLowWaterBytes: 64 * 1024, + spliceHighWaterBytes: 256 * 1024, + // Why: must admit at least one maxFrameBytes frame for a backpressured + // peer, or large catalog responses close the splice with LIMIT_EXCEEDED. + spliceHardQueuedBytes: 8 * 1024 * 1024 + 256 * 1024, + spliceWedgedTimeoutMs: 10 * 1000 +} as const + +export const RELAY_CELL_CONNECTION_HARD_CAPS = [600, 1_000, 3_000] as const +export type RelayCellConnectionHardCap = (typeof RELAY_CELL_CONNECTION_HARD_CAPS)[number] +export const RELAY_CELL_CONNECTION_HARD_CAP: RelayCellConnectionHardCap = 600 + +export function isRelayCellConnectionHardCap( + value: unknown +): value is RelayCellConnectionHardCap { + return RELAY_CELL_CONNECTION_HARD_CAPS.some((hardCap) => hardCap === value) +} + +// Legacy/default bounds remain available for fixed-600 operational gates. +export const RELAY_CELL_ADMISSION_BOUNDS = { + hardCap: RELAY_CELL_CONNECTION_HARD_CAP, + // Ordinary sockets stop here; the remainder is held for same-host control rebinds. + socketAdmissionCeiling: + RELAY_CELL_CONNECTION_HARD_CAP - RELAY_ADMISSION_BUDGETS.reservedHostControls, + // A cell must leave at least one unit admissible after its unobserved allowance. + maxUnobservedBound: + RELAY_CELL_CONNECTION_HARD_CAP - RELAY_ADMISSION_BUDGETS.reservedHostControls - 1 +} as const + +export function relayCellAdmissionBounds(hardCap: RelayCellConnectionHardCap): { + hardCap: RelayCellConnectionHardCap + socketAdmissionCeiling: number + maxUnobservedBound: number +} { + return { + hardCap, + socketAdmissionCeiling: hardCap - RELAY_ADMISSION_BUDGETS.reservedHostControls, + maxUnobservedBound: hardCap - RELAY_ADMISSION_BUDGETS.reservedHostControls - 1 + } +} + +export const RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND = relayCellAdmissionBounds( + RELAY_CELL_CONNECTION_HARD_CAPS.at(-1)! +).maxUnobservedBound + +// Director placement stops short of the socket ceiling by the cell's own unobserved allowance. +export function cellPlacementCeiling( + connectionHardCap: RelayCellConnectionHardCap, + connectionUnobservedBound: number +): number { + return relayCellAdmissionBounds(connectionHardCap).socketAdmissionCeiling - connectionUnobservedBound +} + +export function hasAdmissionCapacity(input: { + totalRequests: number + preAuthConnections: number + sourcePreAuthConnections: number + totalRequestCeiling?: number +}): boolean { + if (input.preAuthConnections >= RELAY_ADMISSION_BUDGETS.maxPreAuthConnections) return false + if (input.sourcePreAuthConnections >= RELAY_ADMISSION_BUDGETS.maxPreAuthPerSource) return false + const totalRequestCeiling = + input.totalRequestCeiling ?? + RELAY_ADMISSION_BUDGETS.cloudRunConcurrency - + RELAY_ADMISSION_BUDGETS.reservedHostControls - + RELAY_ADMISSION_BUDGETS.reservedHostDataSockets + return input.totalRequests < totalRequestCeiling +} diff --git a/cloud/packages/relay-contract/src/assignment-invariants.ts b/cloud/packages/relay-contract/src/assignment-invariants.ts new file mode 100644 index 00000000000..dc60a1ec7b8 --- /dev/null +++ b/cloud/packages/relay-contract/src/assignment-invariants.ts @@ -0,0 +1,56 @@ +import { z } from 'zod' +import { EpochMsSchema, GenerationSchema, OpaqueIdSchema, RelayHostIdSchema } from './wire-scalars.js' + +export const ASSIGNMENT_LIMITS = { + activityLeaseMs: 90 * 1000, + dormantTtlMs: 24 * 60 * 60 * 1000, + migrationLeaseMs: 15 * 60 * 1000 +} as const + +export const AssignmentActivitySchema = z + .object({ + relayHostId: RelayHostIdSchema, + cellId: OpaqueIdSchema, + assignmentEpoch: GenerationSchema, + leaseExpiresAt: EpochMsSchema, + lastActivityAt: EpochMsSchema, + reservedControls: z.number().int().nonnegative(), + reservedSplices: z.number().int().nonnegative(), + reservedInvites: z.number().int().nonnegative(), + pendingInstalls: z.number().int().nonnegative(), + pendingConfirmations: z.number().int().nonnegative(), + migrationLeases: z.number().int().nonnegative() + }) + .strict() + +export function hasAssignmentActivity(record: z.infer): boolean { + return ( + record.reservedControls + + record.reservedSplices + + record.reservedInvites + + record.pendingInstalls + + record.pendingConfirmations + + record.migrationLeases > + 0 + ) +} + +export function mayNormallyReassign( + record: z.infer, + now: number +): boolean { + return !hasAssignmentActivity(record) && now >= record.lastActivityAt + ASSIGNMENT_LIMITS.dormantTtlMs +} + +export const EvacuationCommitSchema = z + .object({ + relayHostId: RelayHostIdSchema, + sourceCellId: OpaqueIdSchema, + targetCellId: OpaqueIdSchema, + previousEpoch: GenerationSchema, + assignmentEpoch: GenerationSchema, + targetCapacityReserved: z.literal(true) + }) + .strict() + .refine((move) => move.sourceCellId !== move.targetCellId, 'target cell must differ') + .refine((move) => move.assignmentEpoch === move.previousEpoch + 1, 'epoch must increment exactly once') diff --git a/cloud/packages/relay-contract/src/close-codes.ts b/cloud/packages/relay-contract/src/close-codes.ts new file mode 100644 index 00000000000..cb2c6307b90 --- /dev/null +++ b/cloud/packages/relay-contract/src/close-codes.ts @@ -0,0 +1,10 @@ +export const RELAY_CLOSE_CODE = { + BAD_OUTER_CREDENTIAL: 4401, + HOST_OFFLINE: 4404, + PEER_DROPPED: 4408, + WRONG_CELL: 4409, + LIMIT_EXCEEDED: 4429, + DRAINING: 4503 +} as const + +export type RelayCloseCode = (typeof RELAY_CLOSE_CODE)[keyof typeof RELAY_CLOSE_CODE] diff --git a/cloud/packages/relay-contract/src/contract.test.ts b/cloud/packages/relay-contract/src/contract.test.ts new file mode 100644 index 00000000000..805cb8ea698 --- /dev/null +++ b/cloud/packages/relay-contract/src/contract.test.ts @@ -0,0 +1,347 @@ +import { describe, expect, it } from 'vitest' +import { RELAY_CLOSE_CODE } from './close-codes.js' +import { + AuthRefreshSchema, + DrainSchema, + HostChallengeSchema, + HostDataAuthSchema, + HostHelloAckSchema, + HostHelloSchema +} from './control-messages.js' +import { + DeviceCredentialInstallSchema, + DeviceResumeConfirmSchema, + RelayAuthSchema, + RelayHelloSchema +} from './credential-messages.js' +import { + AssignmentRequestSchema, + AssignmentResponseSchema, + isTrustedNewerMove, + RelayMovedSchema, + ResolveRequestSchema +} from './director-messages.js' +import { RELAY_PROTOCOL_LIMITS } from './protocol-limits.js' +import { RelayRegionCatalogResponseSchema } from './relay-regions.js' +import { + buildHostChallengePlaintext, + buildHostProofMacInput, + buildHostProofTranscript +} from './host-proof-transcript.js' +import { ConfirmableResumeTupleSchema } from './resume-confirmation-contract.js' +import { + canAdvanceSplice, + mayAcknowledgeClient, + SPLICE_STATE +} from './splice-state-machine.js' + +const TOKEN = 'abcdefghijklmnopqrstuvwxyzABCDEFGH012345678' +const NONCE = 'abcdefghijklmnopqrstuvwxyzABCDEF' +const KEY_B64 = Buffer.alloc(32, 1).toString('base64') + +describe('relay protocol contract', () => { + it('locks close codes and fixed normative limits', () => { + expect(RELAY_CLOSE_CODE).toEqual({ + BAD_OUTER_CREDENTIAL: 4401, + HOST_OFFLINE: 4404, + PEER_DROPPED: 4408, + WRONG_CELL: 4409, + LIMIT_EXCEEDED: 4429, + DRAINING: 4503 + }) + expect(RELAY_PROTOCOL_LIMITS).toMatchObject({ + firstFrameDeadlineMs: 2_000, + maxHttpBodyBytes: 4_096, + maxFrameBytes: 8 * 1024 * 1024, + maxConnectionsPerHost: 8, + idleTimeoutMs: 600_000, + inviteMaxAttempts: 5, + inviteReservationLeaseMs: 15_000, + hostAttachDeadlineMs: 10_000, + resumeConfirmationDeadlineMs: 30_000 + }) + }) + + it('strictly validates host registration and continuity fields', () => { + const hello = { + v: 1, + relayHostId: 'abcdefghijklmnop', + assignmentEpoch: 7, + hostPublicKeyB64: KEY_B64, + appVersion: '1.2.3' + } + expect(HostHelloSchema.safeParse(hello).success).toBe(true) + expect(HostHelloSchema.safeParse({ ...hello, userId: 'injected' }).success).toBe(false) + expect( + HostHelloAckSchema.safeParse({ + v: 1, + generation: 8, + controlResumeSecret: TOKEN, + leaseExpiresAt: 1_800_000_000_000, + activeConnIds: [], + pendingConns: [] + }).success + ).toBe(true) + }) + + it('locks bounded director assignment and resume-only resolve payloads', () => { + const assignment = { v: 1, relayHostId: 'abcdefghijklmnop' } + expect(AssignmentRequestSchema.safeParse(assignment).success).toBe(true) + expect( + AssignmentRequestSchema.safeParse({ ...assignment, preferredRegion: 'asia-east2' }).success + ).toBe(true) + expect( + AssignmentRequestSchema.safeParse({ ...assignment, preferredRegion: 'europe-west1' }).success + ).toBe(false) + expect(AssignmentRequestSchema.safeParse({ ...assignment, relayJwt: 'url-secret' }).success).toBe( + false + ) + expect( + AssignmentResponseSchema.safeParse({ + v: 1, + cellUrl: 'https://relay-c1.onorca.dev', + assignmentEpoch: 3, + lease: 'signed-lease' + }).success + ).toBe(true) + expect( + ResolveRequestSchema.safeParse({ + v: 1, + relayHostId: 'abcdefghijklmnop', + resumeToken: TOKEN + }).success + ).toBe(true) + }) + + it('accepts only unique regions with unique canonical HTTPS probe origins', () => { + const catalog = { + v: 1, + regions: [ + { + region: 'us-central1', + probeOrigins: ['https://relay-c1.example.test', 'https://relay-c2.example.test'] + }, + { region: 'asia-east2', probeOrigins: ['https://relay-c27.example.test'] } + ] + } + expect(RelayRegionCatalogResponseSchema.safeParse(catalog).success).toBe(true) + expect( + RelayRegionCatalogResponseSchema.safeParse({ + ...catalog, + regions: [catalog.regions[0], { ...catalog.regions[0] }] + }).success + ).toBe(false) + expect( + RelayRegionCatalogResponseSchema.safeParse({ + ...catalog, + regions: [ + catalog.regions[0], + { region: 'asia-east2', probeOrigins: ['https://relay-c1.example.test'] } + ] + }).success + ).toBe(false) + for (const origin of [ + 'http://relay-c1.example.test', + 'https://relay-c1.example.test/', + 'https://relay-c1.example.test/health', + 'https://relay-c1.example.test?probe=1' + ]) { + expect( + RelayRegionCatalogResponseSchema.safeParse({ + v: 1, + regions: [{ region: 'us-central1', probeOrigins: [origin] }] + }).success + ).toBe(false) + } + }) + + it('accepts moves only from the configured director at a strictly newer epoch', () => { + const move = RelayMovedSchema.parse({ + v: 1, + cellUrl: 'https://relay-c2.onorca.dev', + assignmentEpoch: 4 + }) + const base = { + configuredDirectorOrigin: 'https://relay.onorca.dev', + currentAssignmentEpoch: 3, + move + } + expect(isTrustedNewerMove({ ...base, sourceOrigin: 'https://relay.onorca.dev' })).toBe(true) + expect(isTrustedNewerMove({ ...base, sourceOrigin: move.cellUrl })).toBe(false) + expect( + isTrustedNewerMove({ + ...base, + sourceOrigin: 'https://relay.onorca.dev', + currentAssignmentEpoch: 4 + }) + ).toBe(false) + }) + + it('bounds challenge material and fixes director-only drain recovery', () => { + expect( + HostChallengeSchema.safeParse({ + challengeId: 'challenge-1', + relayEphemeralPublicKeyB64: KEY_B64, + nonceB64: NONCE, + ciphertextB64: KEY_B64, + expiresAt: 1_800_000_000_000 + }).success + ).toBe(true) + expect(DrainSchema.safeParse({ graceMs: 0, recovery: 'resolve-director' }).success).toBe(true) + expect(DrainSchema.safeParse({ graceMs: 0, recovery: 'follow-server-url' }).success).toBe(false) + expect(AuthRefreshSchema.safeParse({ relayJwt: 'jwt', identity: 'injected' }).success).toBe(false) + }) + + it('strictly binds one-shot host data tickets to a control generation', () => { + expect(HostDataAuthSchema.safeParse({ v: 1, connTicket: TOKEN, generation: 3 }).success).toBe( + true + ) + expect( + HostDataAuthSchema.safeParse({ + v: 1, + connTicket: TOKEN, + generation: 3, + relayDeviceId: 'injected' + }).success + ).toBe(false) + }) + + it('cannot acknowledge a splice before forwarding handlers exist', () => { + expect( + canAdvanceSplice(SPLICE_STATE.PRE_AUTH_ADMITTED, SPLICE_STATE.CREDENTIAL_LEASE_RESERVED) + ).toBe(true) + expect(canAdvanceSplice(SPLICE_STATE.PRE_AUTH_ADMITTED, SPLICE_STATE.SPLICED)).toBe(false) + expect(mayAcknowledgeClient(SPLICE_STATE.HOST_ATTACHED, false)).toBe(false) + expect(mayAcknowledgeClient(SPLICE_STATE.HOST_ATTACHED, true)).toBe(true) + }) + + it('locks the immutable server-owned resume tuple shape', () => { + const tuple = { + basisConnId: 'conn-1', + owningControlGeneration: 4, + relayDeviceId: 'device-1', + acceptedCredentialVersion: 2, + acceptedAs: 'current', + confirmDeadline: 1_800_000_000_000 + } + expect(ConfirmableResumeTupleSchema.safeParse(tuple).success).toBe(true) + expect(ConfirmableResumeTupleSchema.safeParse({ ...tuple, userId: 'caller-value' }).success).toBe( + false + ) + }) + + it('locks the complete host key-possession transcript', () => { + const transcript = buildHostProofTranscript({ + relayOrigin: 'https://relay.onorca.dev', + relayEphemeralPublicKey: new Uint8Array(32).fill(1), + challengeNonce: new Uint8Array(24).fill(4), + challengeId: 'challenge-1', + issuedAt: 1_700_000_000_000, + expiresAt: 1_700_000_010_000, + userId: 'user-1', + profileId: 'profile-1', + organizationId: 'org-1', + relayHostId: 'abcdefghijklmnop', + hostPublicKey: new Uint8Array(32).fill(2), + assignmentEpoch: 7, + previousGeneration: 6, + resumeRequested: true + }) + const challenge = buildHostChallengePlaintext(transcript, new Uint8Array(32).fill(3)) + const proofInput = buildHostProofMacInput(transcript) + + expect(Buffer.from(transcript).toString('base64url')).toBe( + 'AAAACHByb3RvY29sAAAAGG9yY2EtcmVsYXktaG9zdC1wcm9vZi92MQAAAAd2ZXJzaW9uAAAAAQEAAAALcmVsYXlPcmlnaW4AAAAYaHR0cHM6Ly9yZWxheS5vbm9yY2EuZGV2AAAAF3JlbGF5RXBoZW1lcmFsUHVibGljS2V5AAAAIAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAAAADmNoYWxsZW5nZU5vbmNlAAAAGAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAAAAAtjaGFsbGVuZ2VJZAAAAAtjaGFsbGVuZ2UtMQAAAAhpc3N1ZWRBdAAAAAgAAAGLz-VoAAAAAAlleHBpcmVzQXQAAAAIAAABi8_ljxAAAAAGdXNlcklkAAAABnVzZXItMQAAAAlwcm9maWxlSWQAAAAJcHJvZmlsZS0xAAAADm9yZ2FuaXphdGlvbklkAAAABW9yZy0xAAAAC3JlbGF5SG9zdElkAAAAEGFiY2RlZmdoaWprbG1ub3AAAAANaG9zdFB1YmxpY0tleQAAACACAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgAAAA9hc3NpZ25tZW50RXBvY2gAAAAIAAAAAAAAAAcAAAAScHJldmlvdXNHZW5lcmF0aW9uAAAACAAAAAAAAAAGAAAAD3Jlc3VtZVJlcXVlc3RlZAAAAAEB' + ) + expect(challenge.byteLength).toBe(transcript.byteLength + 65) + expect(proofInput.byteLength).toBe(transcript.byteLength + 29) + expect(() => buildHostChallengePlaintext(transcript, new Uint8Array(31))).toThrow( + 'challengeSecret must be 32 bytes' + ) + expect(() => + buildHostProofTranscript({ + relayOrigin: 'https://relay.onorca.dev', + relayEphemeralPublicKey: new Uint8Array(32), + challengeNonce: new Uint8Array(32), + challengeId: 'challenge-1', + issuedAt: 1, + expiresAt: 2, + userId: 'user-1', + profileId: 'profile-1', + organizationId: 'org-1', + relayHostId: 'abcdefghijklmnop', + hostPublicKey: new Uint8Array(32), + assignmentEpoch: 1, + resumeRequested: false + }) + ).toThrow('challengeNonce must be 24 bytes') + }) + + it('accepts only the bounded first-frame credential shape', () => { + expect(RelayAuthSchema.parse({ v: 1, mode: 'connect', credential: TOKEN })).toEqual({ + v: 1, + mode: 'connect', + credential: TOKEN + }) + expect( + RelayAuthSchema.safeParse({ v: 1, mode: 'connect', credential: TOKEN, extra: true }).success + ).toBe(false) + }) + + it('makes install authorization modes mutually exclusive', () => { + const base = { + v: 1, + reqId: 'req-1', + relayDeviceId: 'device-1', + newResumeTokenHash: TOKEN + } + expect( + DeviceCredentialInstallSchema.safeParse({ + ...base, + authorization: { mode: 'relay-basis', basisConnId: 'conn-1' } + }).success + ).toBe(true) + expect( + DeviceCredentialInstallSchema.safeParse({ + ...base, + authorization: { + mode: 'relay-basis', + basisConnId: 'conn-1', + directAuthId: 'injected' + } + }).success + ).toBe(false) + }) + + it('does not let invite attach observations impersonate resume state', () => { + expect( + RelayHelloSchema.safeParse({ + ok: true, + credentialKind: 'invite', + leaseExpiresAt: 1_800_000_000_000 + }).success + ).toBe(true) + expect( + RelayHelloSchema.safeParse({ + ok: true, + credentialKind: 'invite', + leaseExpiresAt: 1_800_000_000_000, + acceptedCredentialVersion: 7, + acceptedAs: 'current', + resumeExpiresAt: 1_800_000_000_000 + }).success + ).toBe(false) + }) + + it('rejects caller-supplied device and credential metadata on resume confirmation', () => { + const confirmation = { v: 1, reqId: 'req-1', basisConnId: 'conn-1' } + expect(DeviceResumeConfirmSchema.safeParse(confirmation).success).toBe(true) + expect( + DeviceResumeConfirmSchema.safeParse({ + ...confirmation, + relayDeviceId: 'injected', + acceptedCredentialVersion: 99 + }).success + ).toBe(false) + }) +}) diff --git a/cloud/packages/relay-contract/src/control-continuity.ts b/cloud/packages/relay-contract/src/control-continuity.ts new file mode 100644 index 00000000000..11821fc537e --- /dev/null +++ b/cloud/packages/relay-contract/src/control-continuity.ts @@ -0,0 +1,43 @@ +export const CONTROL_GENERATION_STATE = { + ACTIVE: 'active', + ORPHANED: 'orphaned', + DRAIN_ONLY: 'drain-only', + FENCED: 'fenced', + CLOSED: 'closed' +} as const + +export const CONTROL_CONTINUITY_LIMITS = { + orphanGraceMs: 30 * 1000, + authRefreshMinBeforeExpiryMs: 60 * 1000, + authRefreshMaxBeforeExpiryMs: 120 * 1000, + expiredAuthExistingSpliceGraceMs: 60 * 1000 +} as const + +export function controlLossDisposition(input: { + matchingResumeSecret: boolean + competingGeneration: boolean +}): 'rebind' | 'fence-old' | 'orphan-grace' { + if (input.matchingResumeSecret) return 'rebind' + if (input.competingGeneration) return 'fence-old' + return 'orphan-grace' +} + +export function mayStartNewRelayWork(state: string, authExpired: boolean): boolean { + return state === CONTROL_GENERATION_STATE.ACTIVE && !authExpired +} + +export interface RelayAuthIdentity { + sub: string + prof: string + org?: string + relayHostId: string +} + +export function preservesRelayAuthIdentity(previous: RelayAuthIdentity, refreshed: RelayAuthIdentity): boolean { + return ( + previous.sub === refreshed.sub && + previous.prof === refreshed.prof && + previous.org === refreshed.org && + previous.relayHostId === refreshed.relayHostId + ) +} diff --git a/cloud/packages/relay-contract/src/control-messages.ts b/cloud/packages/relay-contract/src/control-messages.ts new file mode 100644 index 00000000000..0d5f8d1b851 --- /dev/null +++ b/cloud/packages/relay-contract/src/control-messages.ts @@ -0,0 +1,119 @@ +import { z } from 'zod' +import { + Base6432ByteSchema, + Base64Raw24ByteSchema, + Base64Url32ByteSchema, + EpochMsSchema, + GenerationSchema, + OpaqueIdSchema, + PositiveDurationMsSchema, + RelayHostIdSchema +} from './wire-scalars.js' + +const AppVersionSchema = z.string().min(1).max(128) +const BoundedCiphertextSchema = z + .string() + .min(1) + .max(16 * 1024) + .regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/) +const ConnectionKindSchema = z.enum(['invite', 'resume']) + +export const HostHelloSchema = z + .object({ + v: z.literal(1), + relayHostId: RelayHostIdSchema, + assignmentEpoch: GenerationSchema, + hostPublicKeyB64: Base6432ByteSchema, + appVersion: AppVersionSchema, + previousGeneration: GenerationSchema.optional(), + controlResumeSecret: Base64Url32ByteSchema.optional() + }) + .strict() + +export const HostChallengeSchema = z + .object({ + challengeId: OpaqueIdSchema, + relayEphemeralPublicKeyB64: Base6432ByteSchema, + nonceB64: Base64Raw24ByteSchema, + ciphertextB64: BoundedCiphertextSchema, + expiresAt: EpochMsSchema + }) + .strict() + +export const HostChallengeAckSchema = z + .object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema }) + .strict() + +const PendingConnectionSchema = z + .object({ connId: OpaqueIdSchema, connTicket: Base64Url32ByteSchema }) + .strict() + +export const HostHelloAckSchema = z + .object({ + v: z.literal(1), + generation: GenerationSchema, + controlResumeSecret: Base64Url32ByteSchema, + leaseExpiresAt: EpochMsSchema, + activeConnIds: z.array(OpaqueIdSchema).max(8), + pendingConns: z.array(PendingConnectionSchema).max(8) + }) + .strict() + +export const ConnectionOpenSchema = z + .object({ + connId: OpaqueIdSchema, + connTicket: Base64Url32ByteSchema, + kind: ConnectionKindSchema, + relayDeviceId: OpaqueIdSchema, + attachDeadlineMs: PositiveDurationMsSchema + }) + .strict() + +export const HostDataAuthSchema = z + .object({ + v: z.literal(1), + connTicket: Base64Url32ByteSchema, + generation: GenerationSchema + }) + .strict() + +export const InviteCreateSchema = z + .object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema }) + .strict() + +export const InviteCreatedSchema = z + .object({ + reqId: OpaqueIdSchema, + inviteToken: Base64Url32ByteSchema, + expiresAt: EpochMsSchema, + maxAttempts: z.number().int().positive().max(16) + }) + .strict() + +export const DeviceRevokeSchema = z + .object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema }) + .strict() + +export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict() + +export const DrainSchema = z + .object({ + graceMs: z.number().int().nonnegative().max(60 * 60 * 1000), + recovery: z.literal('resolve-director') + }) + .strict() + +export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict() + +export type HostHello = z.infer +export type HostChallenge = z.infer +export type HostChallengeAck = z.infer +export type HostHelloAck = z.infer +export type ConnectionOpen = z.infer +export type HostDataAuth = z.infer +export type InviteCreate = z.infer +export type InviteCreated = z.infer +export type DeviceRevoke = z.infer +export type AuthRefresh = z.infer +export type Drain = z.infer +export type Heartbeat = z.infer diff --git a/cloud/packages/relay-contract/src/credential-messages.ts b/cloud/packages/relay-contract/src/credential-messages.ts new file mode 100644 index 00000000000..658d2f25f89 --- /dev/null +++ b/cloud/packages/relay-contract/src/credential-messages.ts @@ -0,0 +1,105 @@ +import { z } from 'zod' +import { Base64Url32ByteSchema, EpochMsSchema, OpaqueIdSchema } from './wire-scalars.js' + +export const RelayAuthSchema = z + .object({ v: z.literal(1), mode: z.literal('connect'), credential: Base64Url32ByteSchema }) + .strict() + +const RelayErrorCodeSchema = z.union([ + z.literal(4401), + z.literal(4404), + z.literal(4408), + z.literal(4409), + z.literal(4429), + z.literal(4503) +]) + +export const RelayHelloSchema = z.union([ + z.object({ ok: z.literal(false), code: RelayErrorCodeSchema }).strict(), + z + .object({ + ok: z.literal(true), + credentialKind: z.literal('invite'), + leaseExpiresAt: EpochMsSchema + }) + .strict(), + z + .object({ + ok: z.literal(true), + credentialKind: z.literal('resume'), + leaseExpiresAt: EpochMsSchema, + acceptedCredentialVersion: z.number().int().positive(), + acceptedAs: z.enum(['current', 'grace']), + resumeExpiresAt: EpochMsSchema, + graceExpiresAt: EpochMsSchema.optional() + }) + .strict() +]) + +export const DeviceCredentialInstallSchema = z + .object({ + v: z.literal(1), + reqId: OpaqueIdSchema, + relayDeviceId: OpaqueIdSchema, + newResumeTokenHash: Base64Url32ByteSchema, + expectedCurrentHash: Base64Url32ByteSchema.optional(), + authorization: z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('relay-basis'), basisConnId: OpaqueIdSchema }).strict(), + z.object({ mode: z.literal('authenticated-direct'), directAuthId: OpaqueIdSchema }).strict() + ]) + }) + .strict() + +export const DeviceCredentialInstallStatusSchema = z + .object({ v: z.literal(1), reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema }) + .strict() + +export const DeviceResumeConfirmSchema = z + .object({ v: z.literal(1), reqId: OpaqueIdSchema, basisConnId: OpaqueIdSchema }) + .strict() + +export const DeviceCredentialInstalledSchema = z + .object({ + v: z.literal(1), + reqId: OpaqueIdSchema, + authorizationMode: z.enum(['relay-basis', 'authenticated-direct']), + currentVersion: z.number().int().positive(), + resumeExpiresAt: EpochMsSchema, + graceExpiresAt: EpochMsSchema.optional() + }) + .strict() + +export const DeviceCredentialInstallStatusResultSchema = z.union([ + z.object({ v: z.literal(1), reqId: OpaqueIdSchema, state: z.literal('not-found') }).strict(), + z + .object({ + v: z.literal(1), + reqId: OpaqueIdSchema, + state: z.literal('committed'), + result: DeviceCredentialInstalledSchema + }) + .strict() +]) + +export const DeviceResumeConfirmedSchema = z + .object({ + v: z.literal(1), + reqId: OpaqueIdSchema, + currentVersion: z.number().int().positive(), + acceptedAs: z.enum(['current', 'grace']), + renewed: z.boolean(), + resumeExpiresAt: EpochMsSchema, + graceExpiresAt: EpochMsSchema.optional() + }) + .strict() + +export type RelayAuth = z.infer +export type RelayHello = z.infer +export type DeviceCredentialInstall = z.infer +export type DeviceCredentialInstalled = z.infer +export type DeviceCredentialInstallStatus = z.infer +export type DeviceCredentialInstallStatusResult = z.infer< + typeof DeviceCredentialInstallStatusResultSchema +> +export type DeviceResumeConfirm = z.infer +export type DeviceResumeConfirmed = z.infer diff --git a/cloud/packages/relay-contract/src/director-messages.ts b/cloud/packages/relay-contract/src/director-messages.ts new file mode 100644 index 00000000000..e697135b68e --- /dev/null +++ b/cloud/packages/relay-contract/src/director-messages.ts @@ -0,0 +1,75 @@ +import { z } from 'zod' +import { + Base64Url32ByteSchema, + CanonicalHttpsOriginSchema, + EpochMsSchema, + GenerationSchema, + RelayHostIdSchema +} from './wire-scalars.js' +import { RelayRegionSchema } from './relay-regions.js' + +const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024) + +export const AssignmentRequestSchema = z + .object({ + v: z.literal(1), + relayHostId: RelayHostIdSchema, + // Client-declared reconnection; the director verifies it against the + // durable assignment before granting fast-lane admission. + reconnect: z.boolean().optional(), + preferredRegion: RelayRegionSchema.optional() + }) + .strict() + +export const AssignmentResponseSchema = z + .object({ + v: z.literal(1), + cellUrl: CanonicalHttpsOriginSchema, + assignmentEpoch: GenerationSchema, + lease: SignedAssignmentLeaseSchema + }) + .strict() + +export const ResolveRequestSchema = z + .object({ + v: z.literal(1), + relayHostId: RelayHostIdSchema, + resumeToken: Base64Url32ByteSchema + }) + .strict() + +export const ResolveResponseSchema = z + .object({ + v: z.literal(1), + cellUrl: CanonicalHttpsOriginSchema, + assignmentEpoch: GenerationSchema, + leaseExpiresAt: EpochMsSchema + }) + .strict() + +export const RelayMovedSchema = z + .object({ + v: z.literal(1), + cellUrl: CanonicalHttpsOriginSchema, + assignmentEpoch: GenerationSchema + }) + .strict() + +export function isTrustedNewerMove(input: { + sourceOrigin: string + configuredDirectorOrigin: string + currentAssignmentEpoch: number + move: z.infer +}): boolean { + // Why: cells and stale director responses must never redirect a credential-bearing client. + return ( + input.sourceOrigin === input.configuredDirectorOrigin && + input.move.assignmentEpoch > input.currentAssignmentEpoch + ) +} + +export type AssignmentRequest = z.infer +export type AssignmentResponse = z.infer +export type ResolveRequest = z.infer +export type ResolveResponse = z.infer +export type RelayMoved = z.infer diff --git a/cloud/packages/relay-contract/src/host-proof-transcript.ts b/cloud/packages/relay-contract/src/host-proof-transcript.ts new file mode 100644 index 00000000000..853986f23f2 --- /dev/null +++ b/cloud/packages/relay-contract/src/host-proof-transcript.ts @@ -0,0 +1,103 @@ +const textEncoder = new TextEncoder() + +export const HOST_PROOF_TRANSCRIPT_DOMAIN = 'orca-relay-host-proof/v1' +export const HOST_CHALLENGE_PLAINTEXT_DOMAIN = 'orca-relay-host-challenge/v1' +export const HOST_CHALLENGE_BOX_ALGORITHM = 'Curve25519-XSalsa20-Poly1305' +export const HOST_PROOF_ALGORITHM = 'HMAC-SHA-256' + +export interface HostProofTranscriptInput { + relayOrigin: string + relayEphemeralPublicKey: Uint8Array + challengeNonce: Uint8Array + challengeId: string + issuedAt: number + expiresAt: number + userId: string + profileId: string + organizationId: string + relayHostId: string + hostPublicKey: Uint8Array + assignmentEpoch: number + previousGeneration?: number + resumeRequested: boolean +} + +function uint32(value: number): Uint8Array { + const bytes = new Uint8Array(4) + new DataView(bytes.buffer).setUint32(0, value, false) + return bytes +} + +function uint64(value: number): Uint8Array { + const bytes = new Uint8Array(8) + new DataView(bytes.buffer).setBigUint64(0, BigInt(value), false) + return bytes +} + +function concat(parts: readonly Uint8Array[]): Uint8Array { + const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0)) + let offset = 0 + for (const part of parts) { + output.set(part, offset) + offset += part.byteLength + } + return output +} + +function field(name: string, value: Uint8Array): Uint8Array { + const encodedName = textEncoder.encode(name) + return concat([uint32(encodedName.byteLength), encodedName, uint32(value.byteLength), value]) +} + +function text(value: string): Uint8Array { + return textEncoder.encode(value) +} + +function requireByteLength(value: Uint8Array, expected: number, name: string): void { + if (value.byteLength !== expected) throw new Error(`${name} must be ${expected} bytes`) +} + +export function buildHostProofTranscript(input: HostProofTranscriptInput): Uint8Array { + requireByteLength(input.relayEphemeralPublicKey, 32, 'relayEphemeralPublicKey') + requireByteLength(input.challengeNonce, 24, 'challengeNonce') + requireByteLength(input.hostPublicKey, 32, 'hostPublicKey') + return concat([ + field('protocol', text(HOST_PROOF_TRANSCRIPT_DOMAIN)), + field('version', new Uint8Array([1])), + field('relayOrigin', text(input.relayOrigin)), + field('relayEphemeralPublicKey', input.relayEphemeralPublicKey), + field('challengeNonce', input.challengeNonce), + field('challengeId', text(input.challengeId)), + field('issuedAt', uint64(input.issuedAt)), + field('expiresAt', uint64(input.expiresAt)), + field('userId', text(input.userId)), + field('profileId', text(input.profileId)), + field('organizationId', text(input.organizationId)), + field('relayHostId', text(input.relayHostId)), + field('hostPublicKey', input.hostPublicKey), + field('assignmentEpoch', uint64(input.assignmentEpoch)), + field( + 'previousGeneration', + input.previousGeneration === undefined ? new Uint8Array() : uint64(input.previousGeneration) + ), + field('resumeRequested', new Uint8Array([input.resumeRequested ? 1 : 0])) + ]) +} + +export function buildHostChallengePlaintext( + transcript: Uint8Array, + challengeSecret: Uint8Array +): Uint8Array { + if (challengeSecret.byteLength !== 32) throw new Error('challengeSecret must be 32 bytes') + // Why: the encrypted random secret makes the public transcript insufficient to forge the ack. + return concat([ + text(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`), + uint32(transcript.byteLength), + transcript, + challengeSecret + ]) +} + +export function buildHostProofMacInput(transcript: Uint8Array): Uint8Array { + return concat([text(`${HOST_PROOF_TRANSCRIPT_DOMAIN}\0ack\0`), transcript]) +} diff --git a/cloud/packages/relay-contract/src/index.ts b/cloud/packages/relay-contract/src/index.ts new file mode 100644 index 00000000000..2a7d7d0feda --- /dev/null +++ b/cloud/packages/relay-contract/src/index.ts @@ -0,0 +1,14 @@ +export * from './close-codes.js' +export * from './admission-budgets.js' +export * from './assignment-invariants.js' +export * from './control-messages.js' +export * from './control-continuity.js' +export * from './credential-messages.js' +export * from './director-messages.js' +export * from './host-proof-transcript.js' +export * from './persistence-invariants.js' +export * from './protocol-limits.js' +export * from './resume-confirmation-contract.js' +export * from './relay-regions.js' +export * from './splice-state-machine.js' +export * from './wire-scalars.js' diff --git a/cloud/packages/relay-contract/src/persistence-invariants.test.ts b/cloud/packages/relay-contract/src/persistence-invariants.test.ts new file mode 100644 index 00000000000..291fa7462da --- /dev/null +++ b/cloud/packages/relay-contract/src/persistence-invariants.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from 'vitest' +import { hasAdmissionCapacity, RELAY_ADMISSION_BUDGETS } from './admission-budgets.js' +import { + ASSIGNMENT_LIMITS, + AssignmentActivitySchema, + EvacuationCommitSchema, + hasAssignmentActivity, + mayNormallyReassign +} from './assignment-invariants.js' +import { + CONTROL_GENERATION_STATE, + controlLossDisposition, + mayStartNewRelayWork, + preservesRelayAuthIdentity +} from './control-continuity.js' +import { + CredentialInstallIdentitySchema, + decideResumeCommit, + INSTALL_TRANSACTION_CONTRACT, + INSTALL_STATUS, + InviteRecordSchema, + INVITE_STATE, + nextCredentialVersion, + PAIRING_RECOVERY_MATRIX, + transitionInvite, + TokenFamilySchema +} from './persistence-invariants.js' + +const HASH = 'abcdefghijklmnopqrstuvwxyzABCDEFGH012345678' + +describe('relay persistence invariants', () => { + it('admits pre-auth sockets only outside every reserved budget', () => { + expect(RELAY_ADMISSION_BUDGETS.maxProcessQueuedBytes).toBe(64 * 1024 * 1024) + expect(hasAdmissionCapacity({ totalRequests: 649, preAuthConnections: 44, sourcePreAuthConnections: 3 })).toBe(true) + expect(hasAdmissionCapacity({ totalRequests: 650, preAuthConnections: 0, sourcePreAuthConnections: 0 })).toBe(false) + expect(hasAdmissionCapacity({ totalRequests: 0, preAuthConnections: 45, sourcePreAuthConnections: 0 })).toBe(false) + expect(hasAdmissionCapacity({ totalRequests: 0, preAuthConnections: 0, sourcePreAuthConnections: 4 })).toBe(false) + expect(hasAdmissionCapacity({ + totalRequests: 2_899, + preAuthConnections: 0, + sourcePreAuthConnections: 0, + totalRequestCeiling: 2_900 + })).toBe(true) + expect(hasAdmissionCapacity({ + totalRequests: 2_900, + preAuthConnections: 0, + sourcePreAuthConnections: 0, + totalRequestCeiling: 2_900 + })).toBe(false) + }) + + it('represents persisted invite leases without an intermediate install status', () => { + expect(Object.values(INSTALL_STATUS)).toEqual(['not-found', 'committed']) + expect(INSTALL_TRANSACTION_CONTRACT.atomicEffects).toEqual([ + 'idempotency-result', 'token-family', 'affected-invites' + ]) + expect(nextCredentialVersion(undefined)).toBe(1) + expect(nextCredentialVersion(3)).toBe(4) + expect( + InviteRecordSchema.safeParse({ + relayDeviceId: 'device-1', tokenHash: HASH, state: INVITE_STATE.RESERVED, + attemptCount: 1, maxAttempts: 5, expiresAt: 100, reservationId: 'reservation-1', + reservationExpiresAt: 50 + }).success + ).toBe(true) + expect( + InviteRecordSchema.safeParse({ + relayDeviceId: 'device-1', tokenHash: HASH, state: INVITE_STATE.RESERVED, + attemptCount: 1, maxAttempts: 5, expiresAt: 100 + }).success + ).toBe(false) + expect( + CredentialInstallIdentitySchema.safeParse({ + userId: 'user-1', relayHostId: 'abcdefghijklmnop', relayDeviceId: 'device-1', reqId: 'req-1' + }).success + ).toBe(true) + }) + + it('leases, cools down, rolls back, consumes, invalidates, and cleans invites durably', () => { + const available = InviteRecordSchema.parse({ + relayDeviceId: 'device-1', tokenHash: HASH, state: INVITE_STATE.AVAILABLE, + attemptCount: 0, maxAttempts: 2, expiresAt: 1_000 + }) + const reserved = transitionInvite( + available, + { type: 'reserve', reservationId: 'reservation-1', reservationExpiresAt: 50 }, + 10 + ) + expect(reserved.state).toBe(INVITE_STATE.RESERVED) + expect(transitionInvite(reserved, { type: 'transaction-rollback' }, 20)).toBe(reserved) + const cooldown = transitionInvite(reserved, { type: 'lease-expired', cooldownUntil: 60 }, 50) + expect(cooldown.state).toBe(INVITE_STATE.COOLDOWN) + const second = transitionInvite( + cooldown, + { type: 'reserve', reservationId: 'reservation-2', reservationExpiresAt: 80 }, + 60 + ) + expect(transitionInvite(second, { type: 'attach-failed', cooldownUntil: 90 }, 70).state).toBe( + INVITE_STATE.INVALIDATED + ) + expect(transitionInvite(reserved, { type: 'provision-committed' }, 20).state).toBe( + INVITE_STATE.CONSUMED + ) + expect(transitionInvite(available, { type: 'direct-install-committed' }, 20).state).toBe( + INVITE_STATE.INVALIDATED + ) + expect(transitionInvite(available, { type: 'cleanup' }, 1_000).state).toBe(INVITE_STATE.EXPIRED) + expect(PAIRING_RECOVERY_MATRIX).toHaveLength(4) + }) + + it('makes commit-time current, grace, retired, expired, and revoked outcomes exact', () => { + const family = TokenFamilySchema.parse({ + currentVersion: 3, currentHash: HASH, currentExpiresAt: 200, + graceVersion: 2, graceHash: HASH, graceExpiresAt: 150 + }) + expect(decideResumeCommit(family, 3, 100)).toBe('renew-current') + expect(decideResumeCommit(family, 2, 100)).toBe('return-unchanged-grace') + expect(decideResumeCommit(family, 1, 100)).toBe('reject-retired') + expect(decideResumeCommit(family, 2, 151)).toBe('reject-expired') + expect(decideResumeCommit({ ...family, revokedAt: 90 }, 3, 100)).toBe('reject-revoked') + }) + + it('distinguishes same-process rebind, replacement fencing, and drain-only work', () => { + expect(controlLossDisposition({ matchingResumeSecret: true, competingGeneration: true })).toBe('rebind') + expect(controlLossDisposition({ matchingResumeSecret: false, competingGeneration: true })).toBe('fence-old') + expect(controlLossDisposition({ matchingResumeSecret: false, competingGeneration: false })).toBe('orphan-grace') + expect(mayStartNewRelayWork(CONTROL_GENERATION_STATE.DRAIN_ONLY, false)).toBe(false) + expect(mayStartNewRelayWork(CONTROL_GENERATION_STATE.ACTIVE, true)).toBe(false) + const identity = { sub: 'u', prof: 'p', org: 'o', relayHostId: 'abcdefghijklmnop' } + expect(preservesRelayAuthIdentity(identity, identity)).toBe(true) + expect(preservesRelayAuthIdentity(identity, { ...identity, org: 'other' })).toBe(false) + }) + + it('counts every durable activity class before normal reassignment', () => { + const record = AssignmentActivitySchema.parse({ + relayHostId: 'abcdefghijklmnop', cellId: 'cell-1', assignmentEpoch: 1, + leaseExpiresAt: 100, lastActivityAt: 100, reservedControls: 0, reservedSplices: 0, + reservedInvites: 0, pendingInstalls: 0, pendingConfirmations: 0, migrationLeases: 1 + }) + expect(hasAssignmentActivity(record)).toBe(true) + expect(mayNormallyReassign(record, 100 + ASSIGNMENT_LIMITS.dormantTtlMs)).toBe(false) + expect(mayNormallyReassign({ ...record, migrationLeases: 0 }, 100 + ASSIGNMENT_LIMITS.dormantTtlMs)).toBe(true) + expect( + EvacuationCommitSchema.safeParse({ + relayHostId: 'abcdefghijklmnop', sourceCellId: 'cell-1', targetCellId: 'cell-2', + previousEpoch: 1, assignmentEpoch: 2, targetCapacityReserved: true + }).success + ).toBe(true) + expect( + EvacuationCommitSchema.safeParse({ + relayHostId: 'abcdefghijklmnop', sourceCellId: 'cell-1', targetCellId: 'cell-2', + previousEpoch: 1, assignmentEpoch: 3, targetCapacityReserved: true + }).success + ).toBe(false) + }) +}) diff --git a/cloud/packages/relay-contract/src/persistence-invariants.ts b/cloud/packages/relay-contract/src/persistence-invariants.ts new file mode 100644 index 00000000000..0f07250d008 --- /dev/null +++ b/cloud/packages/relay-contract/src/persistence-invariants.ts @@ -0,0 +1,172 @@ +import { z } from 'zod' +import { Base64Url32ByteSchema, EpochMsSchema, OpaqueIdSchema, RelayHostIdSchema } from './wire-scalars.js' + +export const INVITE_STATE = { + AVAILABLE: 'available', + RESERVED: 'reserved', + COOLDOWN: 'cooldown', + CONSUMED: 'consumed', + EXPIRED: 'expired', + INVALIDATED: 'invalidated' +} as const + +export const InviteRecordSchema = z + .object({ + relayDeviceId: OpaqueIdSchema, + tokenHash: Base64Url32ByteSchema, + state: z.nativeEnum(INVITE_STATE), + attemptCount: z.number().int().nonnegative(), + maxAttempts: z.number().int().positive(), + expiresAt: EpochMsSchema, + reservationId: OpaqueIdSchema.optional(), + reservationExpiresAt: EpochMsSchema.optional(), + cooldownUntil: EpochMsSchema.optional() + }) + .strict() + .superRefine((invite, context) => { + if ( + invite.state === INVITE_STATE.RESERVED && + (!invite.reservationId || !invite.reservationExpiresAt) + ) { + context.addIssue({ code: z.ZodIssueCode.custom, message: 'reserved invite requires a lease' }) + } + if (invite.state === INVITE_STATE.COOLDOWN && invite.cooldownUntil === undefined) { + context.addIssue({ + code: z.ZodIssueCode.custom, + message: 'cooldown invite requires cooldownUntil' + }) + } + }) + +export type InviteRecord = z.infer + +export type InviteEvent = + | { type: 'reserve'; reservationId: string; reservationExpiresAt: number } + | { type: 'attach-failed'; cooldownUntil: number } + | { type: 'lease-expired'; cooldownUntil: number } + | { type: 'provision-committed' } + | { type: 'direct-install-committed' } + | { type: 'transaction-rollback' } + | { type: 'cleanup' } + +function withoutLease(invite: InviteRecord): InviteRecord { + const { reservationId: _reservationId, reservationExpiresAt: _reservationExpiresAt, ...rest } = invite + return rest +} + +export function transitionInvite(invite: InviteRecord, event: InviteEvent, now: number): InviteRecord { + if (event.type === 'transaction-rollback') return invite + if (event.type === 'direct-install-committed') { + return { ...withoutLease(invite), state: INVITE_STATE.INVALIDATED } + } + if (event.type === 'provision-committed') { + return { ...withoutLease(invite), state: INVITE_STATE.CONSUMED } + } + if (event.type === 'cleanup') { + if (now >= invite.expiresAt) return { ...withoutLease(invite), state: INVITE_STATE.EXPIRED } + if ( + invite.state === INVITE_STATE.RESERVED && + invite.reservationExpiresAt !== undefined && + now >= invite.reservationExpiresAt + ) { + const state = + invite.attemptCount >= invite.maxAttempts ? INVITE_STATE.INVALIDATED : INVITE_STATE.COOLDOWN + return { ...withoutLease(invite), state, cooldownUntil: now } + } + return invite + } + if (event.type === 'reserve') { + const available = + invite.state === INVITE_STATE.AVAILABLE || + (invite.state === INVITE_STATE.COOLDOWN && (invite.cooldownUntil ?? 0) <= now) + if (!available || now >= invite.expiresAt || invite.attemptCount >= invite.maxAttempts) return invite + return { + ...invite, + state: INVITE_STATE.RESERVED, + attemptCount: invite.attemptCount + 1, + reservationId: event.reservationId, + reservationExpiresAt: event.reservationExpiresAt, + cooldownUntil: undefined + } + } + if (invite.state !== INVITE_STATE.RESERVED) return invite + const state = + invite.attemptCount >= invite.maxAttempts ? INVITE_STATE.INVALIDATED : INVITE_STATE.COOLDOWN + return { ...withoutLease(invite), state, cooldownUntil: event.cooldownUntil } +} + +export const PAIRING_RECOVERY_MATRIX = [ + 'direct-commit-ack', + 'direct-commit-response-lost', + 'direct-no-commit-invite-fallback', + 'late-direct-versus-invite-global-key' +] as const + +export const CredentialInstallIdentitySchema = z + .object({ + userId: OpaqueIdSchema, + relayHostId: RelayHostIdSchema, + relayDeviceId: OpaqueIdSchema, + reqId: OpaqueIdSchema + }) + .strict() + +export const INSTALL_STATUS = { + NOT_FOUND: 'not-found', + COMMITTED: 'committed' +} as const + +export const INSTALL_TRANSACTION_CONTRACT = { + idempotencyKeyFields: ['userId', 'relayHostId', 'relayDeviceId', 'reqId'], + transactionDeadlineMs: 10 * 1000, + lockAttemptTimeoutMs: 2 * 1000, + maxDeadlockRetries: 3, + atomicEffects: ['idempotency-result', 'token-family', 'affected-invites'] +} as const + +export function nextCredentialVersion(currentVersion: number | undefined): number { + return currentVersion === undefined ? 1 : currentVersion + 1 +} + +export const TokenFamilySchema = z + .object({ + currentVersion: z.number().int().positive(), + currentHash: Base64Url32ByteSchema, + currentExpiresAt: EpochMsSchema, + graceVersion: z.number().int().positive().optional(), + graceHash: Base64Url32ByteSchema.optional(), + graceExpiresAt: EpochMsSchema.optional(), + revokedAt: EpochMsSchema.optional() + }) + .strict() + .superRefine((family, context) => { + const graceFields = [family.graceVersion, family.graceHash, family.graceExpiresAt] + if (graceFields.some((value) => value !== undefined) && graceFields.some((value) => value === undefined)) { + context.addIssue({ code: z.ZodIssueCode.custom, message: 'grace fields must be all present or absent' }) + } + if (family.graceVersion !== undefined && family.graceVersion >= family.currentVersion) { + context.addIssue({ code: z.ZodIssueCode.custom, message: 'grace version must precede current' }) + } + }) + +export type ResumeCommitDecision = + | 'renew-current' + | 'return-unchanged-grace' + | 'reject-retired' + | 'reject-expired' + | 'reject-revoked' + +export function decideResumeCommit( + family: z.infer, + acceptedVersion: number, + now: number +): ResumeCommitDecision { + if (family.revokedAt !== undefined && family.revokedAt <= now) return 'reject-revoked' + if (acceptedVersion === family.currentVersion) { + return family.currentExpiresAt > now ? 'renew-current' : 'reject-expired' + } + if (acceptedVersion === family.graceVersion) { + return (family.graceExpiresAt ?? 0) > now ? 'return-unchanged-grace' : 'reject-expired' + } + return 'reject-retired' +} diff --git a/cloud/packages/relay-contract/src/protocol-limits.ts b/cloud/packages/relay-contract/src/protocol-limits.ts new file mode 100644 index 00000000000..6257f9cecb7 --- /dev/null +++ b/cloud/packages/relay-contract/src/protocol-limits.ts @@ -0,0 +1,23 @@ +export const RELAY_PROTOCOL_LIMITS = { + firstFrameDeadlineMs: 2_000, + maxHttpBodyBytes: 4 * 1024, + // Why: the splice is an opaque E2EE stream; the desktop's worktree catalog + // response already exceeds 1MiB on large workspaces (~775KiB at 415 + // worktrees, growing), and an oversized frame kills the session on every + // reconnect. 8MiB buys years of headroom; catalog pagination is the + // long-term fix on the desktop side. + maxFrameBytes: 8 * 1024 * 1024, + maxConnectionsPerHost: 8, + idleTimeoutMs: 10 * 60 * 1000, + inviteTtlMs: 10 * 60 * 1000, + inviteMaxAttempts: 5, + inviteReservationLeaseMs: 15 * 1000, + inviteAttemptCooldownMs: 2 * 1000, + hostAttachDeadlineMs: 10 * 1000, + resumeConfirmationDeadlineMs: 30 * 1000, + resumeTtlMs: 30 * 24 * 60 * 60 * 1000, + relayTokenTtlMs: 5 * 60 * 1000, + expiredAuthExistingSpliceGraceMs: 60 * 1000, + controlPingIntervalMs: 15 * 1000, + controlSilenceTimeoutMs: 75 * 1000 +} as const diff --git a/cloud/packages/relay-contract/src/relay-regions.ts b/cloud/packages/relay-contract/src/relay-regions.ts new file mode 100644 index 00000000000..38ac36cd738 --- /dev/null +++ b/cloud/packages/relay-contract/src/relay-regions.ts @@ -0,0 +1,62 @@ +import { z } from 'zod' + +export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const + +export const RelayRegionSchema = z.enum(RELAY_REGIONS) + +export type RelayRegion = z.infer + +export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1' + +const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin) + +export const RelayRegionCatalogResponseSchema = z + .object({ + v: z.literal(1), + regions: z + .array( + z + .object({ + region: RelayRegionSchema, + probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2) + }) + .strict() + ) + .max(RELAY_REGIONS.length) + }) + .strict() + .superRefine((catalog, context) => { + const regions = new Set() + const origins = new Set() + for (const [regionIndex, entry] of catalog.regions.entries()) { + if (regions.has(entry.region)) { + context.addIssue({ + code: 'custom', + message: 'duplicate relay region', + path: ['regions', regionIndex, 'region'] + }) + } + regions.add(entry.region) + for (const [originIndex, origin] of entry.probeOrigins.entries()) { + if (origins.has(origin)) { + context.addIssue({ + code: 'custom', + message: 'duplicate relay probe origin', + path: ['regions', regionIndex, 'probeOrigins', originIndex] + }) + } + origins.add(origin) + } + } + }) + +export type RelayRegionCatalogResponse = z.infer + +function isCanonicalHttpsOrigin(value: string): boolean { + try { + const url = new URL(value) + return url.protocol === 'https:' && url.origin === value + } catch { + return false + } +} diff --git a/cloud/packages/relay-contract/src/resume-confirmation-contract.ts b/cloud/packages/relay-contract/src/resume-confirmation-contract.ts new file mode 100644 index 00000000000..0f429fc9e81 --- /dev/null +++ b/cloud/packages/relay-contract/src/resume-confirmation-contract.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import { EpochMsSchema, GenerationSchema, OpaqueIdSchema } from './wire-scalars.js' + +export const ConfirmableResumeTupleSchema = z + .object({ + basisConnId: OpaqueIdSchema, + owningControlGeneration: GenerationSchema, + relayDeviceId: OpaqueIdSchema, + acceptedCredentialVersion: z.number().int().positive(), + acceptedAs: z.enum(['current', 'grace']), + confirmDeadline: EpochMsSchema + }) + .strict() + +export const RESUME_CONFIRMATION_COMMIT_OUTCOME = { + RENEW_CURRENT: 'renew-current', + RETURN_UNCHANGED_GRACE: 'return-unchanged-grace', + REJECT_RETIRED: 'reject-retired', + REJECT_EXPIRED: 'reject-expired', + REJECT_REVOKED: 'reject-revoked' +} as const + +export type ConfirmableResumeTuple = z.infer diff --git a/cloud/packages/relay-contract/src/splice-state-machine.ts b/cloud/packages/relay-contract/src/splice-state-machine.ts new file mode 100644 index 00000000000..3944df74584 --- /dev/null +++ b/cloud/packages/relay-contract/src/splice-state-machine.ts @@ -0,0 +1,34 @@ +export const SPLICE_STATE = { + PRE_AUTH_ADMITTED: 'pre-auth-admitted', + CREDENTIAL_LEASE_RESERVED: 'credential-lease-reserved', + HOST_NOTIFIED: 'host-notified', + ATTACH_PENDING: 'attach-pending', + HOST_ATTACHED: 'host-attached', + CLIENT_ACKNOWLEDGED: 'client-acknowledged', + SPLICED: 'spliced', + E2EE_CONFIRMABLE: 'e2ee-confirmable', + TEARDOWN: 'teardown' +} as const + +export type SpliceState = (typeof SPLICE_STATE)[keyof typeof SPLICE_STATE] + +export const SPLICE_FORWARD_TRANSITIONS: Readonly> = { + [SPLICE_STATE.PRE_AUTH_ADMITTED]: [SPLICE_STATE.CREDENTIAL_LEASE_RESERVED, SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.CREDENTIAL_LEASE_RESERVED]: [SPLICE_STATE.HOST_NOTIFIED, SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.HOST_NOTIFIED]: [SPLICE_STATE.ATTACH_PENDING, SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.ATTACH_PENDING]: [SPLICE_STATE.HOST_ATTACHED, SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.HOST_ATTACHED]: [SPLICE_STATE.CLIENT_ACKNOWLEDGED, SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.CLIENT_ACKNOWLEDGED]: [SPLICE_STATE.SPLICED, SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.SPLICED]: [SPLICE_STATE.E2EE_CONFIRMABLE, SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.E2EE_CONFIRMABLE]: [SPLICE_STATE.TEARDOWN], + [SPLICE_STATE.TEARDOWN]: [] +} + +export function canAdvanceSplice(from: SpliceState, to: SpliceState): boolean { + return SPLICE_FORWARD_TRANSITIONS[from].includes(to) +} + +export function mayAcknowledgeClient(state: SpliceState, forwardingHandlersInstalled: boolean): boolean { + // Why: success before both forwarding handlers exist can strand a client on a fake splice. + return state === SPLICE_STATE.HOST_ATTACHED && forwardingHandlersInstalled +} diff --git a/cloud/packages/relay-contract/src/wire-scalars.ts b/cloud/packages/relay-contract/src/wire-scalars.ts new file mode 100644 index 00000000000..27dd3a8b30f --- /dev/null +++ b/cloud/packages/relay-contract/src/wire-scalars.ts @@ -0,0 +1,20 @@ +import { z } from 'zod' + +export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/) +export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/) +export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/) +export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/) +export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/) +export const OpaqueIdSchema = z.string().min(1).max(128) +export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000) + +export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => { + try { + const url = new URL(value) + return url.protocol === 'https:' && url.origin === value && url.pathname === '/' + } catch { + return false + } +}, 'must be a canonical HTTPS origin') diff --git a/cloud/packages/relay-contract/tsconfig.build.json b/cloud/packages/relay-contract/tsconfig.build.json new file mode 100644 index 00000000000..94c84b60803 --- /dev/null +++ b/cloud/packages/relay-contract/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": false, + "noEmit": false, + "outDir": "dist", + "rootDir": "src" + }, + "exclude": ["src/**/*.test.ts"] +} diff --git a/cloud/packages/relay-contract/tsconfig.json b/cloud/packages/relay-contract/tsconfig.json new file mode 100644 index 00000000000..a552e34dbe9 --- /dev/null +++ b/cloud/packages/relay-contract/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true }, + "include": ["src/**/*.ts"] +} diff --git a/cloud/pnpm-lock.yaml b/cloud/pnpm-lock.yaml new file mode 100644 index 00000000000..27fdd29071a --- /dev/null +++ b/cloud/pnpm-lock.yaml @@ -0,0 +1,1336 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + tsx: + specifier: ^4.21.0 + version: 4.22.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + + apps/relay: + dependencies: + '@hono/node-server': + specifier: ^1.19.14 + version: 1.19.14(hono@4.12.27) + '@orca-cloud/relay-contract': + specifier: workspace:* + version: link:../../packages/relay-contract + hono: + specifier: ^4.12.27 + version: 4.12.27 + jose: + specifier: ^6.1.3 + version: 6.2.3 + pg: + specifier: ^8.22.0 + version: 8.22.0 + tweetnacl: + specifier: ^1.0.3 + version: 1.0.3 + ws: + specifier: ^8.18.3 + version: 8.21.0 + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + '@types/pg': + specifier: ^8.20.0 + version: 8.20.0 + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 + tsx: + specifier: ^4.21.0 + version: 4.22.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + + apps/relay-fence-broker: + dependencies: + '@hono/node-server': + specifier: ^1.19.14 + version: 1.19.14(hono@4.12.27) + hono: + specifier: ^4.12.27 + version: 4.12.27 + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + tsx: + specifier: ^4.21.0 + version: 4.22.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + + apps/relay-ops: + dependencies: + '@hono/node-server': + specifier: ^1.19.14 + version: 1.19.14(hono@4.12.27) + hono: + specifier: ^4.12.27 + version: 4.12.27 + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + tsx: + specifier: ^4.21.0 + version: 4.22.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + + packages/relay-contract: + dependencies: + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^24.10.0 + version: 24.13.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.8 + version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + +packages: + + '@emnapi/core@1.10.0': + resolution: {integrity: sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==} + + '@emnapi/runtime@1.10.0': + resolution: {integrity: sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==} + + '@emnapi/wasi-threads@1.2.1': + resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@hono/node-server@1.19.14': + resolution: {integrity: sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==} + engines: {node: '>=18.14.1'} + peerDependencies: + hono: ^4 + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@napi-rs/wasm-runtime@1.1.5': + resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + + '@oxc-project/types@0.133.0': + resolution: {integrity: sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==} + + '@rolldown/binding-android-arm64@1.0.3': + resolution: {integrity: sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.0.3': + resolution: {integrity: sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.0.3': + resolution: {integrity: sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.0.3': + resolution: {integrity: sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.0.3': + resolution: {integrity: sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.0.3': + resolution: {integrity: sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + + '@rolldown/binding-linux-arm64-musl@1.0.3': + resolution: {integrity: sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + + '@rolldown/binding-linux-ppc64-gnu@1.0.3': + resolution: {integrity: sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + + '@rolldown/binding-linux-s390x-gnu@1.0.3': + resolution: {integrity: sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + + '@rolldown/binding-linux-x64-gnu@1.0.3': + resolution: {integrity: sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + + '@rolldown/binding-linux-x64-musl@1.0.3': + resolution: {integrity: sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + + '@rolldown/binding-openharmony-arm64@1.0.3': + resolution: {integrity: sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-wasm32-wasi@1.0.3': + resolution: {integrity: sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + + '@rolldown/binding-win32-arm64-msvc@1.0.3': + resolution: {integrity: sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.0.3': + resolution: {integrity: sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@tybys/wasm-util@0.10.2': + resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/node@24.13.2': + resolution: {integrity: sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==} + + '@types/pg@8.20.0': + resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==} + + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + + '@vitest/expect@4.1.9': + resolution: {integrity: sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==} + + '@vitest/mocker@4.1.9': + resolution: {integrity: sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.1.9': + resolution: {integrity: sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==} + + '@vitest/runner@4.1.9': + resolution: {integrity: sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==} + + '@vitest/snapshot@4.1.9': + resolution: {integrity: sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==} + + '@vitest/spy@4.1.9': + resolution: {integrity: sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==} + + '@vitest/utils@4.1.9': + resolution: {integrity: sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + es-module-lexer@2.1.0: + resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + hono@4.12.27: + resolution: {integrity: sha512-1yrb/+w6HWQJrUCLkJ2IF5jNIPvvFkblV5RNOYl6bV+OA6p9GLcMpHFFGTosSvHvcAUibuUukRqhlYI4z32C7Q==} + engines: {node: '>=16.9.0'} + + jose@6.2.3: + resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} + + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + nanoid@3.3.13: + resolution: {integrity: sha512-sPdqC6ByMVVGvF1ynvvMo0/o+oD1VX7DaHhijt1bFgjvBkHBib4t49GoNDhf2NDta4oeUNlaGbSt5K7qjZ955Q==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + obug@2.1.3: + resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==} + engines: {node: '>=12.20.0'} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.15.0: + resolution: {integrity: sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.22.0: + resolution: {integrity: sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.4: + resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} + engines: {node: '>=12'} + + postcss@8.5.15: + resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} + engines: {node: ^10 || ^12 || >=14} + + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + rolldown@1.0.3: + resolution: {integrity: sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@4.1.0: + resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@1.2.4: + resolution: {integrity: sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==} + engines: {node: '>=18'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinyrainbow@3.1.0: + resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} + engines: {node: '>=14.0.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.22.4: + resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} + engines: {node: '>=18.0.0'} + hasBin: true + + tweetnacl@1.0.3: + resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + + vite@8.0.16: + resolution: {integrity: sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.1.18 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.1.9: + resolution: {integrity: sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.9 + '@vitest/browser-preview': 4.1.9 + '@vitest/browser-webdriverio': 4.1.9 + '@vitest/coverage-istanbul': 4.1.9 + '@vitest/coverage-v8': 4.1.9 + '@vitest/ui': 4.1.9 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + +snapshots: + + '@emnapi/core@1.10.0': + dependencies: + '@emnapi/wasi-threads': 1.2.1 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.10.0': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.2.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@hono/node-server@1.19.14(hono@4.12.27)': + dependencies: + hono: 4.12.27 + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@napi-rs/wasm-runtime@1.1.5(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@tybys/wasm-util': 0.10.2 + optional: true + + '@oxc-project/types@0.133.0': {} + + '@rolldown/binding-android-arm64@1.0.3': + optional: true + + '@rolldown/binding-darwin-arm64@1.0.3': + optional: true + + '@rolldown/binding-darwin-x64@1.0.3': + optional: true + + '@rolldown/binding-freebsd-x64@1.0.3': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.0.3': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.0.3': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.0.3': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.0.3': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.0.3': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.0.3': + optional: true + + '@rolldown/binding-linux-x64-musl@1.0.3': + optional: true + + '@rolldown/binding-openharmony-arm64@1.0.3': + optional: true + + '@rolldown/binding-wasm32-wasi@1.0.3': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@napi-rs/wasm-runtime': 1.1.5(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.0.3': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.0.3': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + + '@standard-schema/spec@1.1.0': {} + + '@tybys/wasm-util@0.10.2': + dependencies: + tslib: 2.8.1 + optional: true + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.9': {} + + '@types/node@24.13.2': + dependencies: + undici-types: 7.18.2 + + '@types/pg@8.20.0': + dependencies: + '@types/node': 24.13.2 + pg-protocol: 1.15.0 + pg-types: 2.2.0 + + '@types/ws@8.18.1': + dependencies: + '@types/node': 24.13.2 + + '@vitest/expect@4.1.9': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.1.9 + '@vitest/utils': 4.1.9 + chai: 6.2.2 + tinyrainbow: 3.1.0 + + '@vitest/mocker@4.1.9(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))': + dependencies: + '@vitest/spy': 4.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4) + + '@vitest/pretty-format@4.1.9': + dependencies: + tinyrainbow: 3.1.0 + + '@vitest/runner@4.1.9': + dependencies: + '@vitest/utils': 4.1.9 + pathe: 2.0.3 + + '@vitest/snapshot@4.1.9': + dependencies: + '@vitest/pretty-format': 4.1.9 + '@vitest/utils': 4.1.9 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.1.9': {} + + '@vitest/utils@4.1.9': + dependencies: + '@vitest/pretty-format': 4.1.9 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 + + assertion-error@2.0.1: {} + + chai@6.2.2: {} + + convert-source-map@2.0.0: {} + + detect-libc@2.1.2: {} + + es-module-lexer@2.1.0: {} + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.3.0: {} + + fdir@6.5.0(picomatch@4.0.4): + optionalDependencies: + picomatch: 4.0.4 + + fsevents@2.3.3: + optional: true + + hono@4.12.27: {} + + jose@6.2.3: {} + + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + nanoid@3.3.13: {} + + obug@2.1.3: {} + + pathe@2.0.3: {} + + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-int8@1.0.1: {} + + pg-pool@3.14.0(pg@8.22.0): + dependencies: + pg: 8.22.0 + + pg-protocol@1.15.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.22.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.22.0) + pg-protocol: 1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + + picocolors@1.1.1: {} + + picomatch@4.0.4: {} + + postcss@8.5.15: + dependencies: + nanoid: 3.3.13 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + rolldown@1.0.3: + dependencies: + '@oxc-project/types': 0.133.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm64': 1.0.3 + '@rolldown/binding-darwin-arm64': 1.0.3 + '@rolldown/binding-darwin-x64': 1.0.3 + '@rolldown/binding-freebsd-x64': 1.0.3 + '@rolldown/binding-linux-arm-gnueabihf': 1.0.3 + '@rolldown/binding-linux-arm64-gnu': 1.0.3 + '@rolldown/binding-linux-arm64-musl': 1.0.3 + '@rolldown/binding-linux-ppc64-gnu': 1.0.3 + '@rolldown/binding-linux-s390x-gnu': 1.0.3 + '@rolldown/binding-linux-x64-gnu': 1.0.3 + '@rolldown/binding-linux-x64-musl': 1.0.3 + '@rolldown/binding-openharmony-arm64': 1.0.3 + '@rolldown/binding-wasm32-wasi': 1.0.3 + '@rolldown/binding-win32-arm64-msvc': 1.0.3 + '@rolldown/binding-win32-x64-msvc': 1.0.3 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + split2@4.2.0: {} + + stackback@0.0.2: {} + + std-env@4.1.0: {} + + tinybench@2.9.0: {} + + tinyexec@1.2.4: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.4) + picomatch: 4.0.4 + + tinyrainbow@3.1.0: {} + + tslib@2.8.1: + optional: true + + tsx@4.22.4: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + + tweetnacl@1.0.3: {} + + typescript@5.9.3: {} + + undici-types@7.18.2: {} + + vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4): + dependencies: + lightningcss: 1.32.0 + picomatch: 4.0.4 + postcss: 8.5.15 + rolldown: 1.0.3 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 24.13.2 + esbuild: 0.28.1 + fsevents: 2.3.3 + tsx: 4.22.4 + + vitest@4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)): + dependencies: + '@vitest/expect': 4.1.9 + '@vitest/mocker': 4.1.9(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)) + '@vitest/pretty-format': 4.1.9 + '@vitest/runner': 4.1.9 + '@vitest/snapshot': 4.1.9 + '@vitest/spy': 4.1.9 + '@vitest/utils': 4.1.9 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.3 + pathe: 2.0.3 + picomatch: 4.0.4 + std-env: 4.1.0 + tinybench: 2.9.0 + tinyexec: 1.2.4 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.0 + vite: 8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 24.13.2 + transitivePeerDependencies: + - msw + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + ws@8.21.0: {} + + xtend@4.0.2: {} + + zod@3.25.76: {} diff --git a/cloud/pnpm-workspace.yaml b/cloud/pnpm-workspace.yaml new file mode 100644 index 00000000000..cc61e60464c --- /dev/null +++ b/cloud/pnpm-workspace.yaml @@ -0,0 +1,4 @@ +packages: + - apps/* + - packages/* + diff --git a/cloud/tsconfig.base.json b/cloud/tsconfig.base.json new file mode 100644 index 00000000000..d977ad5c9b9 --- /dev/null +++ b/cloud/tsconfig.base.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "declaration": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "lib": ["ES2024"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noUncheckedIndexedAccess": true, + "outDir": "dist", + "resolveJsonModule": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2024" + } +} + diff --git a/config/oxlint-react-doctor.json b/config/oxlint-react-doctor.json index 3c91b01d58d..c39f0d51bf2 100644 --- a/config/oxlint-react-doctor.json +++ b/config/oxlint-react-doctor.json @@ -25,5 +25,5 @@ "react-doctor/zustand-no-mutating-state": "warn", "react-doctor/zustand-no-whole-store-destructure": "warn" }, - "ignorePatterns": ["**/node_modules", "**/dist", "**/out"] + "ignorePatterns": ["**/node_modules", "**/dist", "**/out", "cloud/**"] } diff --git a/config/scripts/check-changed-code-quality.mjs b/config/scripts/check-changed-code-quality.mjs index 4427c6b7f38..eedf3dbda78 100644 --- a/config/scripts/check-changed-code-quality.mjs +++ b/config/scripts/check-changed-code-quality.mjs @@ -7,6 +7,7 @@ import { resolvePullRequestDiffBase } from './git-pull-request-diff-base.mjs' import { resolveOxlintInvocation } from './oxlint-cli-invocation.mjs' const SOURCE_FILE_PATTERN = /\.(?:[cm]?[jt]sx?)$/ +const ROOT_CODE_QUALITY_IGNORED_PREFIXES = ['cloud/'] export const OXLINT_SCANS = [ { // Why: no --config, so Oxlint keeps discovering nested configs. Pinning the root @@ -73,6 +74,10 @@ function splitNullDelimited(output) { return output.split('\0').filter(Boolean) } +export function isRootCodeQualityPath(file) { + return !ROOT_CODE_QUALITY_IGNORED_PREFIXES.some((prefix) => file.startsWith(prefix)) +} + function resolveBase(root, requestedBase) { for (const candidate of [ requestedBase, @@ -107,7 +112,11 @@ export function collectAddedLineRanges(root, requestedBase) { const rangesByFile = new Map() for (const file of changedFiles) { - if (!SOURCE_FILE_PATTERN.test(file) || !existsSync(path.join(root, file))) { + if ( + !isRootCodeQualityPath(file) || + !SOURCE_FILE_PATTERN.test(file) || + !existsSync(path.join(root, file)) + ) { continue } const diff = runGit(root, ['diff', '--unified=0', '--no-color', comparisonBase, '--', file]) @@ -119,7 +128,11 @@ export function collectAddedLineRanges(root, requestedBase) { for (const file of untrackedFiles) { const absolutePath = path.join(root, file) - if (!SOURCE_FILE_PATTERN.test(file) || !existsSync(absolutePath)) { + if ( + !isRootCodeQualityPath(file) || + !SOURCE_FILE_PATTERN.test(file) || + !existsSync(absolutePath) + ) { continue } const lineCount = readFileSync(absolutePath, 'utf8').split(/\r?\n/).length diff --git a/config/scripts/check-changed-code-quality.test.mjs b/config/scripts/check-changed-code-quality.test.mjs index 76a25802e5c..3a88cf1b02e 100644 --- a/config/scripts/check-changed-code-quality.test.mjs +++ b/config/scripts/check-changed-code-quality.test.mjs @@ -3,6 +3,7 @@ import { OXLINT_SCANS, diagnosticTouchesAddedLines, isMovedCode, + isRootCodeQualityPath, overlapsAddedLines, parseAddedLineRanges } from './check-changed-code-quality.mjs' @@ -52,6 +53,11 @@ describe('changed-code quality line matching', () => { expect(scan.args).not.toContain('--config') expect(scan.args).not.toContain('--disable-nested-config') }) + + it('leaves Cloud source to the independent Cloud quality checks', () => { + expect(isRootCodeQualityPath('cloud/apps/relay/src/index.ts')).toBe(false) + expect(isRootCodeQualityPath('src/main/index.ts')).toBe(true) + }) }) describe('moved-code exemption', () => { diff --git a/config/scripts/check-root-directory-entries.test.mjs b/config/scripts/check-root-directory-entries.test.mjs index 15276e8aa82..dcc5596771b 100644 --- a/config/scripts/check-root-directory-entries.test.mjs +++ b/config/scripts/check-root-directory-entries.test.mjs @@ -105,6 +105,15 @@ describe('root directory guard', () => { expect(output).toContain('new-root.md') }) + it('allows the reviewed cloud workspace directory', () => { + const fixture = makeFixture() + const head = commitFiles(fixture.root, [['cloud/package.json', '{}\n']]) + + const result = runGuard({ ...fixture, head }) + + expect(result.status).toBe(0) + }) + it('rejects a new top-level directory', () => { const fixture = makeFixture() const head = commitFiles(fixture.root, [['new-folder/file.txt', 'too prominent\n']]) diff --git a/config/scripts/pr-code-change-scope.mjs b/config/scripts/pr-code-change-scope.mjs index 67d4f565afa..8bc10fc5b72 100644 --- a/config/scripts/pr-code-change-scope.mjs +++ b/config/scripts/pr-code-change-scope.mjs @@ -237,6 +237,8 @@ const WINDOWS_PACKAGE_TESTS = [ const DESKTOP_IRRELEVANT_PREFIXES = [ 'mobile/', + 'cloud/', + '.github/workflows/cloud-', '.github/workflows/mobile.yml', '.github/workflows/mobile-ios-release.yml', '.github/workflows/mobile-android-release.yml' diff --git a/config/scripts/pr-code-change-scope.test.mjs b/config/scripts/pr-code-change-scope.test.mjs index 9a1c9e649b6..1fe296af265 100644 --- a/config/scripts/pr-code-change-scope.test.mjs +++ b/config/scripts/pr-code-change-scope.test.mjs @@ -86,6 +86,17 @@ describe('docs-only path classification', () => { it('does not start desktop PR Checks for mobile-only diffs', () => { expect(shouldRunPrChecks(['mobile/src/App.tsx', 'mobile/package.json'])).toBe(false) }) + + it('does not start desktop PR Checks for cloud-only diffs', () => { + expect( + shouldRunPrChecks([ + 'cloud/apps/relay/src/index.ts', + 'cloud/package.json', + 'cloud/.gitleaks.toml', + '.github/workflows/cloud-verify.yml' + ]) + ).toBe(false) + }) }) describe('per-job path classification', () => { diff --git a/package.json b/package.json index 005a1e12b0d..179ffd1a82a 100644 --- a/package.json +++ b/package.json @@ -293,12 +293,12 @@ "windows-native-registry": "3.2.2" }, "lint-staged": { - "*.{ts,tsx,js,jsx,mjs,mts,cts}": [ + "{*.{ts,tsx,js,jsx,mjs,mts,cts},!(cloud)/**/*.{ts,tsx,js,jsx,mjs,mts,cts}}": [ "oxlint", "oxlint --config config/oxlint-react-doctor.json", "oxfmt --write" ], - "*.{json,css}": [ + "{*.{json,css},!(cloud)/**/*.{json,css}}": [ "oxfmt --write" ] },