diff --git a/.github/workflows/cloud-deploy-relay-production-director.yml b/.github/workflows/cloud-deploy-relay-production-director.yml index 6094f25ccaf..d517fd27347 100644 --- a/.github/workflows/cloud-deploy-relay-production-director.yml +++ b/.github/workflows/cloud-deploy-relay-production-director.yml @@ -51,6 +51,10 @@ on: description: Exact immutable serving predecessor digest for the one-time identity bootstrap required: true type: string + shape-baseline-revision: + description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts + required: false + type: string permissions: contents: read @@ -93,6 +97,7 @@ jobs: EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }} PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} + SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }} steps: - uses: actions/checkout@v4 @@ -140,7 +145,13 @@ jobs: [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false test "${REGIONAL_PLACEMENT_MODE}" = preserve + test -z "${SHAPE_BASELINE_REVISION}" test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY + elif test -n "${SHAPE_BASELINE_REVISION}"; then + [[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-[0-9]{5}-[a-z0-9]{3}$ ]] + test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false + test "${REGIONAL_PLACEMENT_MODE}" = preserve + test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then test "${REGIONAL_PLACEMENT_MODE}" = preserve test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS @@ -222,12 +233,17 @@ jobs: fi [[ "${target_version}" =~ ^[1-9][0-9]*$ ]] RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" + shape_args=() + if test -n "${SHAPE_BASELINE_REVISION}"; then + shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}") + fi node dev/scripts/deploy-relay-blue-green.mjs \ --project "${GCP_PROJECT_ID}" \ --region "${GCP_REGION}" \ --service "${DIRECTOR_SERVICE_NAME}" \ --image "${IMAGE}" \ --role director \ + "${shape_args[@]}" \ --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ --predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \ --bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \ diff --git a/.github/workflows/cloud-deploy-relay-staging.yml b/.github/workflows/cloud-deploy-relay-staging.yml index 93ef7acd264..bf24f6bab8d 100644 --- a/.github/workflows/cloud-deploy-relay-staging.yml +++ b/.github/workflows/cloud-deploy-relay-staging.yml @@ -39,7 +39,11 @@ on: type: choice options: [preserve, '0', '2'] confirmation: - description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap + description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap, ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE with a shape baseline + required: false + type: string + shape-baseline-revision: + description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts required: false type: string shadow-seat-feed-cells: @@ -86,6 +90,7 @@ jobs: PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }} MIN_INSTANCES: ${{ inputs.min-instances }} + SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }} steps: - uses: actions/checkout@v4 @@ -106,6 +111,12 @@ jobs: else test -z "${PREDECESSOR_IMAGE_DIGEST}" fi + if test -n "${SHAPE_BASELINE_REVISION}"; then + [[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-staging-[0-9]{5}-[a-z0-9]{3}$ ]] + test "${BOOTSTRAP_RUNTIME_IDENTITY}" = false + test "${CAPACITY_CELL_ID}" = none + test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE + fi - uses: hashicorp/setup-terraform@v3 with: @@ -186,6 +197,10 @@ jobs: if test "${MIN_INSTANCES}" != preserve; then floor_args=(--min-instances "${MIN_INSTANCES}") fi + shape_args=() + if test -n "${SHAPE_BASELINE_REVISION}"; then + shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}") + fi topology_args=() if test "${CAPACITY_CELL_ID}" != none; then topology_args=( @@ -207,6 +222,7 @@ jobs: "${floor_args[@]}" \ --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ "${identity_args[@]}" \ + "${shape_args[@]}" \ --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ --rehome-audience "${REHOME_AUDIENCE}" \ --rehome-control-origin https://relay-staging.onorca.dev \ diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index 55ef3f894ad..0266bd53add 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -142,6 +142,7 @@ "google_logging_metric.relay_cell_reserve_dead_man_tripped", "google_logging_metric.relay_control_flags_unreadable", "google_logging_metric.relay_incident", + "google_logging_metric.relay_event_loop_stall", "google_logging_metric.relay_long_cell_inventory_hold", "google_logging_metric.relay_reserve_placement_off_with_reserve_cells", "google_logging_metric.relay_snapshot", @@ -167,6 +168,7 @@ "google_monitoring_alert_policy.relay_far_cell_accept_latency", "google_monitoring_alert_policy.relay_gce_connection_headroom", "google_monitoring_alert_policy.relay_lock_timeout_burst", + "google_monitoring_alert_policy.relay_event_loop_stall", "google_monitoring_alert_policy.relay_long_cell_inventory_hold", "google_monitoring_alert_policy.relay_postgres_retry_exhausted", "google_monitoring_alert_policy.relay_process_fatal", @@ -216,6 +218,7 @@ "google_secret_manager_secret.push_provider", "google_secret_manager_secret.relay_assignment_signing_key", "google_secret_manager_secret.relay_database_url", + "google_secret_manager_secret.relay_director_database_url", "google_secret_manager_secret.relay_regional_placement_enabled", "google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor", "google_secret_manager_secret_iam_member.push_provider_runtime_accessor", @@ -223,6 +226,7 @@ "google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor", "google_secret_manager_secret_iam_member.relay_database_url_accessor", "google_secret_manager_secret_iam_member.relay_database_url_director_accessor", + "google_secret_manager_secret_iam_member.relay_director_database_url_accessor", "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor", "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder", "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer", @@ -231,6 +235,7 @@ "google_secret_manager_secret_version.push_dedicated_database_url", "google_secret_manager_secret_version.relay_assignment_signing_key", "google_secret_manager_secret_version.relay_database_url", + "google_secret_manager_secret_version.relay_director_database_url", "google_secret_manager_secret_version.relay_regional_placement_enabled", "google_service_account.github_fence", "google_service_account.github_monitor", diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs index d93b2617e86..08a18475cab 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs @@ -377,6 +377,28 @@ export function parseArguments(argv) { if (!['true', 'false'].includes(values['bootstrap-runtime-identity'] ?? 'false')) { throw new Error('--bootstrap-runtime-identity must be true or false') } + if ( + values['shape-baseline-revision'] !== undefined && + !/^[a-z][a-z0-9-]{0,62}$/.test(values['shape-baseline-revision']) + ) { + throw new Error('--shape-baseline-revision must be a Cloud Run revision name') + } + if (values['shape-baseline-revision'] !== undefined) { + if (values.role !== 'director') { + throw new Error('--shape-baseline-revision requires --role director') + } + // A shape adoption moves only what the reviewed apply changed: no identity, topology or + // pruning rides along, whichever caller builds the arguments. + if (values['bootstrap-runtime-identity'] === 'true') { + throw new Error('--shape-baseline-revision cannot combine with a runtime identity bootstrap') + } + if (values['capacity-cell-id'] !== undefined || values['director-cells-json'] !== undefined) { + throw new Error('--shape-baseline-revision cannot combine with a director topology change') + } + if (values['prune-revisions'] === 'true') { + throw new Error('--shape-baseline-revision cannot combine with revision pruning') + } + } if (values.role === 'director') { directorDeploymentEnvironment(values) projectServiceAccount(values, 'runtime-service-account') @@ -896,6 +918,31 @@ function assertDirectorRevisionIdentity(revision, config) { } } +// The reviewed shape a deploy must preserve. Normally the serving revision's; after a Terraform +// apply changes the template (a sidecar, a volume, a secret reference), the apply's own 0% +// revision, named by the operator, is the reviewed shape instead. +export function directorShapeBaseline(config, service, servingRevision, describe) { + const named = config['shape-baseline-revision'] + if (named === undefined) return servingRevision + if (named === activeRevision(service)) { + throw new Error('shape baseline revision is already serving; deploy without it') + } + if (named !== service.status?.latestCreatedRevisionName) { + throw new Error('shape baseline revision is not the latest created revision') + } + const servingImage = servingRevision.spec?.containers?.[0]?.image + // A shape change and an image change never ride together: each is reviewed on its own. + if (config.image?.split('@').at(-1) !== servingImage?.split('@').at(-1)) { + throw new Error('a shape baseline deploy must keep the serving image') + } + const baseline = describe(config, named) + // Terraform ignores the image, so its revision runs what is serving; anything else was not an apply. + if (baseline.spec?.containers?.[0]?.image !== servingImage) { + throw new Error('shape baseline revision does not run the serving image') + } + return baseline +} + export async function deployDirector(config, tag, overrides = {}) { const operations = { deployCandidate, @@ -916,6 +963,12 @@ export async function deployDirector(config, tag, overrides = {}) { // restating it here; public admission is per-instance, so losing it shrinks fleet capacity. const initialService = operations.describeService(config) const servingRevision = operations.describeRevision(config, activeRevision(initialService)) + const shapeBaseline = directorShapeBaseline( + config, + initialService, + servingRevision, + operations.describeRevision + ) const bootstrapRuntimeIdentity = config['bootstrap-runtime-identity'] === 'true' if (bootstrapRuntimeIdentity) { if ( @@ -1023,7 +1076,7 @@ export async function deployDirector(config, tag, overrides = {}) { const rollbackRevision = operations.describeRevision(config, rollback.revision) assertDirectorRevisionIdentity(rollbackRevision, config) assertPreservedRevisionShape( - servingRevision, + shapeBaseline, rollbackRevision, mutableEnvironment, bootstrapRuntimeIdentity @@ -1058,7 +1111,7 @@ export async function deployDirector(config, tag, overrides = {}) { const candidateRevision = operations.describeRevision(config, candidate.revision) assertDirectorRevisionIdentity(candidateRevision, config) assertPreservedRevisionShape( - servingRevision, + shapeBaseline, candidateRevision, mutableEnvironment, bootstrapRuntimeIdentity diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs index 4ab531b58a7..0ba40b6f97b 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs @@ -441,7 +441,9 @@ function directorHarness({ reserveCells = {}, servingEnvironment = {}, placements = {}, - callerEmail + callerEmail, + // The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one. + templateTransport } = {}) { const state = { activeRevision: 'relay-00001-old', @@ -468,7 +470,9 @@ function directorHarness({ } ] ]), - nextRevision: 2 + nextRevision: 2, + latestCreated: 'relay-00001-old', + templateTransport } const removed = [] const healthProtocols = [] @@ -477,6 +481,7 @@ function directorHarness({ describeService: () => ({ status: { url: 'https://relay-hash-uc.a.run.app', + latestCreatedRevisionName: state.latestCreated, traffic: [ { percent: 100, revisionName: state.activeRevision }, ...[...state.tags].map(([tag, revisionName]) => ({ @@ -493,13 +498,15 @@ function directorHarness({ metadata: { annotations: { 'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0), - 'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5) + 'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5), + ...value?.transport?.annotations } }, spec: { serviceAccountName: value?.serviceAccount, containers: [ { + name: value?.transport?.name, image: value?.image, env: [ ...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })), @@ -508,7 +515,8 @@ function directorHarness({ valueSource: { secretKeyRef } })) ] - } + }, + ...value?.transport?.sidecars ?? [] ] } } @@ -529,8 +537,10 @@ function directorHarness({ serviceAccount: config['runtime-service-account'] ?? state.revisions.get(state.activeRevision)?.serviceAccount, - image: image ?? state.revisions.get(state.activeRevision)?.image + image: image ?? state.revisions.get(state.activeRevision)?.image, + transport: state.templateTransport }) + state.latestCreated = revision state.tags.set(tag, revision) if (deployFailure) throw deployFailure }, @@ -715,6 +725,105 @@ test('director deploy refuses to move traffic onto a candidate that lost the flo assert.equal(harness.state.activeRevision, 'relay-00001-old') }) +const SERVING_IMAGE = `relay@sha256:${'f'.repeat(64)}` + +// What a Terraform apply of relay_director_cloud_sql_proxy_sidecar puts in the template. +const SIDECAR_TRANSPORT = { + name: 'relay', + annotations: { 'run.googleapis.com/container-dependencies': '{"relay":["cloud-sql-proxy"]}' }, + sidecars: [{ name: 'cloud-sql-proxy', image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:1' }] +} + +// The apply leaves its revision at 0%, as the director service declares no traffic block. +function applySidecarTemplate(harness) { + const serving = harness.state.revisions.get(harness.state.activeRevision) + harness.state.templateTransport = SIDECAR_TRANSPORT + harness.state.revisions.set('relay-00009-terraform', { ...serving, transport: SIDECAR_TRANSPORT }) + harness.state.latestCreated = 'relay-00009-terraform' +} + +test('a deploy after a sidecar apply refuses the new shape unless the apply revision is named', async () => { + const harness = directorHarness() + applySidecarTemplate(harness) + await assert.rejects( + deployDirector({}, 'candidate-new', harness.operations), + /unrelated revision shape/ + ) + assert.equal(harness.state.activeRevision, 'relay-00001-old') +}) + +test('a deploy adopts the shape of the named Terraform apply revision', async () => { + const harness = directorHarness() + applySidecarTemplate(harness) + await deployDirector( + { image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, + 'candidate-new', + harness.operations + ) + const active = harness.state.revisions.get(harness.state.activeRevision) + assert.equal(active.transport, SIDECAR_TRANSPORT) + assert.equal( + harness.state.revisions.get(harness.state.tags.get('selector-rollback')).transport, + SIDECAR_TRANSPORT + ) +}) + +test('the named shape baseline must be the unserved latest revision on the serving image', async () => { + const stale = directorHarness() + applySidecarTemplate(stale) + await assert.rejects( + deployDirector({ 'shape-baseline-revision': 'relay-00000-stale' }, 'candidate-new', stale.operations), + /not the latest created revision/ + ) + const serving = directorHarness() + await assert.rejects( + deployDirector({ 'shape-baseline-revision': 'relay-00001-old' }, 'candidate-new', serving.operations), + /already serving/ + ) + const otherImage = directorHarness() + applySidecarTemplate(otherImage) + otherImage.state.revisions.get('relay-00009-terraform').image = `relay@sha256:${'e'.repeat(64)}` + await assert.rejects( + deployDirector({ image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, 'candidate-new', otherImage.operations), + /does not run the serving image/ + ) + const newImage = directorHarness() + applySidecarTemplate(newImage) + await assert.rejects( + deployDirector( + { image: `relay@sha256:${'e'.repeat(64)}`, 'shape-baseline-revision': 'relay-00009-terraform' }, + 'candidate-new', + newImage.operations + ), + /must keep the serving image/ + ) + for (const harness of [stale, serving, otherImage, newImage]) { + assert.equal(harness.state.activeRevision, 'relay-00001-old') + } +}) + +test('--shape-baseline-revision takes only a revision name, and only for a director', () => { + const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', 'i@sha256:' + 'a'.repeat(64), '--release-id', 'x'] + assert.throws( + () => parseArguments([...base, '--role', 'director', '--shape-baseline-revision', 'Bad_Name']), + /must be a Cloud Run revision name/ + ) + assert.throws( + () => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--shape-baseline-revision', 'relay-1']), + /--shape-baseline-revision requires --role director/ + ) + const director = [...base, '--role', 'director', '--shape-baseline-revision', 'relay-1'] + assert.equal(parseArguments(director)['shape-baseline-revision'], 'relay-1') + for (const [extra, refusal] of [ + [['--bootstrap-runtime-identity', 'true'], /cannot combine with a runtime identity bootstrap/], + [['--capacity-cell-id', 'c1', '--director-cells-json', '[]'], /cannot combine with a director topology change/], + [['--director-cells-json', '[]'], /cannot combine with a director topology change/], + [['--prune-revisions', 'true'], /cannot combine with revision pruning/] + ]) { + assert.throws(() => parseArguments([...director, ...extra]), refusal) + } +}) + test('director deploy rejects unrelated revision-shape drift', async () => { const harness = directorHarness() const describeRevision = harness.operations.describeRevision diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars index d26a0ee23f5..942b6a05342 100644 --- a/cloud/infra/terraform/environments/staging.tfvars +++ b/cloud/infra/terraform/environments/staging.tfvars @@ -18,6 +18,14 @@ relay_staging_power_auth_service_name = "orca-cloud-auth-staging" relay_base_url = "https://relay-staging.onorca.dev" relay_min_instances = 0 relay_max_instances = 2 + +# Staging proves these before production turns them on. +relay_director_cloud_sql_proxy_sidecar = true +relay_event_loop_stall_alert = true +# What the serving staging director runs (revision 00171, 2026-10-11). Pinned so the sidecar apply, +# whose revision the next deploy adopts as its shape, does not also drop it to the 80 / 30 s defaults. +relay_director_concurrency = 1000 +relay_director_request_timeout_seconds = 3600 # Staging now exercises the production-shaped GCE data plane exclusively. relay_cells = {} # Keep the stable director mapping Terraform-owned while removing cell mappings. diff --git a/cloud/infra/terraform/relay-database.tf b/cloud/infra/terraform/relay-database.tf index 5dbbd71aa31..6e3e0f1a527 100644 --- a/cloud/infra/terraform/relay-database.tf +++ b/cloud/infra/terraform/relay-database.tf @@ -39,6 +39,40 @@ resource "google_secret_manager_secret_version" "relay_database_url" { ) } +# The director's URL when it reaches Cloud SQL through its proxy sidecar. Cells keep the socket URL. +resource "google_secret_manager_secret" "relay_director_database_url" { + count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 + + project = var.project_id + secret_id = "orca-cloud-relay-director-database-url" + labels = local.relay_shared_labels + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "relay_director_database_url" { + count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 + + secret = google_secret_manager_secret.relay_director_database_url[0].id + secret_data = format( + "postgresql://%s:%s@127.0.0.1:5432/%s", + google_sql_user.relay.name, + random_password.relay_database.result, + google_sql_database.relay.name + ) +} + +resource "google_secret_manager_secret_iam_member" "relay_director_database_url_accessor" { + count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 + + project = var.project_id + secret_id = google_secret_manager_secret.relay_director_database_url[0].secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_director_runtime.member +} + resource "google_secret_manager_secret_iam_member" "relay_database_url_accessor" { project = var.project_id secret_id = google_secret_manager_secret.relay_database_url.secret_id diff --git a/cloud/infra/terraform/relay-observability.tf b/cloud/infra/terraform/relay-observability.tf index e1f99cef4fc..5609d223a7a 100644 --- a/cloud/infra/terraform/relay-observability.tf +++ b/cloud/infra/terraform/relay-observability.tf @@ -850,6 +850,86 @@ resource "google_monitoring_alert_policy" "relay_cell_process_exit" { depends_on = [google_logging_metric.relay_incident] } +# One count per runtime sample whose interval held a 5 s or longer freeze. A count, not a distribution: +# a percentile of the max would hide a single freeze among healthy samples. Gated until staging +# has proved it (variables.tf). +resource "google_logging_metric" "relay_event_loop_stall" { + count = var.relay_event_loop_stall_alert ? 1 : 0 + + project = var.project_id + name = "orca_relay_event_loop_stalls" + description = "Runtime samples whose interval held an event-loop freeze of at least 5,000 ms." + filter = "${local.relay_runtime_log_filter} AND jsonPayload.eventLoopDelayMsMax>=5000" + label_extractors = { + role = "EXTRACT(jsonPayload.role)" + cell_id = "EXTRACT(jsonPayload.cellId)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + + labels { + key = "role" + value_type = "STRING" + description = "Relay process role." + } + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier, or director." + } + } +} + +resource "google_monitoring_alert_policy" "relay_event_loop_stall" { + count = var.relay_event_loop_stall_alert ? 1 : 0 + + project = var.project_id + display_name = "Orca Relay: event loop stall" + combiner = "OR" + enabled = true + # Not paging until staging has shown its noise floor. + notification_channels = [] + + dynamic "conditions" { + for_each = { + "Cloud Run" = "cloud_run_revision" + "GCE cell" = "gce_instance" + } + + content { + display_name = "Event loop froze at least 5,000 ms (${conditions.key})" + + condition_threshold { + # The metric counts only samples at or over 5,000 ms, so any count is a breach. + filter = "resource.type=\"${conditions.value}\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_event_loop_stall[0].name}\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "60s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["metric.label.\"role\"", "metric.label.\"cell_id\""] + } + + trigger { + count = 1 + } + } + } + } + + documentation { + content = "A relay process froze its event loop for 5 s or more. While frozen it serves nothing, polls nothing and no timeout fires, so assigns hang and reserve cells lose their dead-man feed. On a director, look for Cloud SQL connection failures in the same window: a connect on Cloud Run's built-in /cloudsql socket blocks the loop until the connector's dial ends (staging 2026-10-11, production 2026-09-28). Otherwise look for a long synchronous task." + mime_type = "text/markdown" + } +} + # Pages on cell holds only: director holds of 1-2.5 s recur several times a day with rehoming paused, # and pausing rehome does not stop them, so paging on them would ask on-call to do nothing. resource "google_monitoring_alert_policy" "relay_long_cell_inventory_hold" { diff --git a/cloud/infra/terraform/relay.tf b/cloud/infra/terraform/relay.tf index 217f92cfd47..ae7994c5bfc 100644 --- a/cloud/infra/terraform/relay.tf +++ b/cloud/infra/terraform/relay.tf @@ -147,16 +147,25 @@ resource "google_cloud_run_v2_service" "relay" { max_instance_count = var.relay_max_instances } - volumes { - name = "cloudsql" + # Built-in socket only without the sidecar: its connect() blocks the event loop (variables.tf). + dynamic "volumes" { + for_each = var.relay_director_cloud_sql_proxy_sidecar ? [] : ["cloudsql"] - cloud_sql_instance { - instances = [local.relay_database_connection_name] + content { + name = volumes.value + + cloud_sql_instance { + instances = [local.relay_database_connection_name] + } } } + # Stays first: deploys and readers address the relay as containers[0], and gcloud flags without + # --container land on the ingress container. containers { - image = var.relay_cloud_run_image + name = var.relay_director_cloud_sql_proxy_sidecar ? "relay" : null + image = var.relay_cloud_run_image + depends_on = var.relay_director_cloud_sql_proxy_sidecar ? ["cloud-sql-proxy"] : null env { name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED" @@ -194,9 +203,13 @@ resource "google_cloud_run_v2_service" "relay" { container_port = 8080 } - volume_mounts { - name = "cloudsql" - mount_path = "/cloudsql" + dynamic "volume_mounts" { + for_each = var.relay_director_cloud_sql_proxy_sidecar ? [] : ["cloudsql"] + + content { + name = volume_mounts.value + mount_path = "/cloudsql" + } } env { @@ -204,7 +217,9 @@ resource "google_cloud_run_v2_service" "relay" { value_source { secret_key_ref { - secret = google_secret_manager_secret.relay_database_url.secret_id + secret = (var.relay_director_cloud_sql_proxy_sidecar + ? google_secret_manager_secret.relay_director_database_url[0].secret_id + : google_secret_manager_secret.relay_database_url.secret_id) version = "latest" } } @@ -380,6 +395,75 @@ resource "google_cloud_run_v2_service" "relay" { cpu_idle = false } } + + # Accepts a loopback TCP connection at once and dials Cloud SQL in the background, so the pool's + # 2 s connect timeout holds through a start-up or failover. Authenticates as the director's + # runtime account. The GCE cells run the same proxy, over a Unix socket on a full Linux kernel. + dynamic "containers" { + for_each = var.relay_director_cloud_sql_proxy_sidecar ? ["cloud-sql-proxy"] : [] + + content { + name = containers.value + image = var.relay_gce_cloud_sql_proxy_image + args = [ + "--address=127.0.0.1", + "--port=5432", + "--health-check", + # Cloud Run's probe is not documented to reach loopback; only the ingress port is routed. + "--http-address=0.0.0.0", + "--http-port=9090", + "--structured-logs", + "--exit-zero-on-sigterm", + # Lets in-flight statements finish while the relay drains after SIGTERM. + "--max-sigterm-delay=10s", + local.relay_database_connection_name + ] + + # Go sizes its scheduler to the host's cores, not the 0.25 vCPU quota; one keeps it from + # being throttled in bursts. + env { + name = "GOMAXPROCS" + value = "1" + } + + # Ready once it listens; it does not wait for the database, so the relay boots through + # an outage exactly as it does today. + startup_probe { + period_seconds = 1 + timeout_seconds = 1 + failure_threshold = 60 + + http_get { + path = "/startup" + port = 9090 + } + } + + # /liveness answers without the database; failing it means the proxy itself is wedged. + liveness_probe { + period_seconds = 10 + timeout_seconds = 1 + failure_threshold = 3 + + http_get { + path = "/liveness" + port = 9090 + } + } + + # A quarter vCPU carries the pool's few connections. Cloud Run rejects only a total under + # 1 vCPU with concurrency above 1 (validated against the API with the relay's 1 vCPU and + # concurrency 1000), so the sidecar adds 25% to director compute, not 100%. + resources { + limits = { + cpu = "0.25" + memory = "256Mi" + } + + cpu_idle = false + } + } + } } # Deploys update immutable images; Terraform owns service shape and IAM. @@ -407,7 +491,9 @@ resource "google_cloud_run_v2_service" "relay" { google_secret_manager_secret_iam_member.relay_database_url_director_accessor, google_secret_manager_secret_version.relay_assignment_signing_key, google_secret_manager_secret_version.relay_regional_placement_enabled, - google_secret_manager_secret_version.relay_database_url + google_secret_manager_secret_version.relay_database_url, + google_secret_manager_secret_iam_member.relay_director_database_url_accessor, + google_secret_manager_secret_version.relay_director_database_url ] } diff --git a/cloud/infra/terraform/variables.tf b/cloud/infra/terraform/variables.tf index 6be05489c3f..a9ae1b7d923 100644 --- a/cloud/infra/terraform/variables.tf +++ b/cloud/infra/terraform/variables.tf @@ -480,9 +480,21 @@ variable "relay_cloud_sql_private_ip" { default = false } +variable "relay_director_cloud_sql_proxy_sidecar" { + type = bool + description = "Reach Cloud SQL from the director through a Cloud SQL Auth Proxy sidecar on 127.0.0.1 (the GCE cells run the same proxy beside the relay, over a Unix socket), instead of Cloud Run's built-in /cloudsql socket. A connect() on the built-in socket blocks the director's whole event loop until the connector's upstream dial ends, up to 10 s per attempt while the instance starts or fails over." + default = false +} + +variable "relay_event_loop_stall_alert" { + type = bool + description = "Count runtime samples whose event-loop delay max reached 5 s and alert on any. The p99 metric reads normal through a single long freeze." + default = false +} + variable "relay_gce_cloud_sql_proxy_image" { type = string - description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers." + description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers and the director sidecar." default = "gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:fc224915ef435afeb5b2a9421260a0d31986d5c8b7c7f5783c7f5d5885700cd2" validation {