From 2bdb55350d37c9d65763660e9876b3fdeb5ad8ce Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:30:27 -0400 Subject: [PATCH] fix(relay): director reaches Cloud SQL through a proxy sidecar (staging), adoptable deploy shape, event-loop stall alert (#27406) * fix(relay): director reaches Cloud SQL through a proxy sidecar (staging), gated event-loop stall alert On Cloud Run's built-in /cloudsql socket a connect() blocks the director's whole event loop until the connector's upstream dial ends, 10 s per attempt while the instance starts. Staging froze both directors for 134 s on 2026-10-11 and production four directors for 40-50 s on 2026-09-28. relay_director_cloud_sql_proxy_sidecar runs the Cloud SQL Auth Proxy beside the director on 127.0.0.1:5432, as the GCE cells already do, with its own TCP database URL secret. relay_event_loop_stall_alert adds the event_loop_ms_max metric and a 5 s stall alert. Both default off, so production plans unchanged; staging turns both on. * fix(relay): adopt a reviewed Terraform director shape on deploy; counter-based stall alert; cheaper probed sidecar - deploy-relay-blue-green: --shape-baseline-revision names the 0% revision a Terraform apply created; the deploy preserves that shape instead of the serving one. It must be the latest created, unserved revision on the serving image. Wired into the staging and production director workflows behind ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE. - Event-loop stall alert is now a gated counter metric on eventLoopDelayMsMax>=5000. - Sidecar gets a /liveness probe and 0.25 vCPU (Cloud Run rejects only a total under 1 vCPU with concurrency above 1; checked with validateOnly against the API). - staging.tfvars pins the serving director's concurrency 1000 and timeout 3600 s, so the adopted apply shape does not fall to the 80 / 30 s defaults. * fix(relay): shape adoption refuses identity, topology, pruning and image changes; GOMAXPROCS=1 on the sidecar parseArguments now refuses --shape-baseline-revision with a runtime identity bootstrap, a director topology change or revision pruning, so every caller is covered, and the deploy refuses a shape baseline whose image differs from the serving one. Both director workflows also refuse the combinations up front. The proxy sidecar runs with GOMAXPROCS=1 to match its 0.25 vCPU quota. The long-hold alert comment is back above its resource. --- ...cloud-deploy-relay-production-director.yml | 16 +++ .../workflows/cloud-deploy-relay-staging.yml | 18 ++- .../terraform-root-partition/families.json | 5 + cloud/dev/scripts/deploy-relay-blue-green.mjs | 57 ++++++++- .../scripts/deploy-relay-blue-green.test.mjs | 119 +++++++++++++++++- .../terraform/environments/staging.tfvars | 8 ++ cloud/infra/terraform/relay-database.tf | 34 +++++ cloud/infra/terraform/relay-observability.tf | 80 ++++++++++++ cloud/infra/terraform/relay.tf | 106 ++++++++++++++-- cloud/infra/terraform/variables.tf | 14 ++- 10 files changed, 438 insertions(+), 19 deletions(-) diff --git a/.github/workflows/cloud-deploy-relay-production-director.yml b/.github/workflows/cloud-deploy-relay-production-director.yml index 6094f25ccaf..d517fd27347 100644 --- a/.github/workflows/cloud-deploy-relay-production-director.yml +++ b/.github/workflows/cloud-deploy-relay-production-director.yml @@ -51,6 +51,10 @@ on: description: Exact immutable serving predecessor digest for the one-time identity bootstrap required: true type: string + shape-baseline-revision: + description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts + required: false + type: string permissions: contents: read @@ -93,6 +97,7 @@ jobs: EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }} PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} + SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }} steps: - uses: actions/checkout@v4 @@ -140,7 +145,13 @@ jobs: [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false test "${REGIONAL_PLACEMENT_MODE}" = preserve + test -z "${SHAPE_BASELINE_REVISION}" test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY + elif test -n "${SHAPE_BASELINE_REVISION}"; then + [[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-[0-9]{5}-[a-z0-9]{3}$ ]] + test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false + test "${REGIONAL_PLACEMENT_MODE}" = preserve + test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then test "${REGIONAL_PLACEMENT_MODE}" = preserve test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS @@ -222,12 +233,17 @@ jobs: fi [[ "${target_version}" =~ ^[1-9][0-9]*$ ]] RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" + shape_args=() + if test -n "${SHAPE_BASELINE_REVISION}"; then + shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}") + fi node dev/scripts/deploy-relay-blue-green.mjs \ --project "${GCP_PROJECT_ID}" \ --region "${GCP_REGION}" \ --service "${DIRECTOR_SERVICE_NAME}" \ --image "${IMAGE}" \ --role director \ + "${shape_args[@]}" \ --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ --predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \ --bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \ diff --git a/.github/workflows/cloud-deploy-relay-staging.yml b/.github/workflows/cloud-deploy-relay-staging.yml index 93ef7acd264..bf24f6bab8d 100644 --- a/.github/workflows/cloud-deploy-relay-staging.yml +++ b/.github/workflows/cloud-deploy-relay-staging.yml @@ -39,7 +39,11 @@ on: type: choice options: [preserve, '0', '2'] confirmation: - description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap + description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap, ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE with a shape baseline + required: false + type: string + shape-baseline-revision: + description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts required: false type: string shadow-seat-feed-cells: @@ -86,6 +90,7 @@ jobs: PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }} MIN_INSTANCES: ${{ inputs.min-instances }} + SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }} steps: - uses: actions/checkout@v4 @@ -106,6 +111,12 @@ jobs: else test -z "${PREDECESSOR_IMAGE_DIGEST}" fi + if test -n "${SHAPE_BASELINE_REVISION}"; then + [[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-staging-[0-9]{5}-[a-z0-9]{3}$ ]] + test "${BOOTSTRAP_RUNTIME_IDENTITY}" = false + test "${CAPACITY_CELL_ID}" = none + test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE + fi - uses: hashicorp/setup-terraform@v3 with: @@ -186,6 +197,10 @@ jobs: if test "${MIN_INSTANCES}" != preserve; then floor_args=(--min-instances "${MIN_INSTANCES}") fi + shape_args=() + if test -n "${SHAPE_BASELINE_REVISION}"; then + shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}") + fi topology_args=() if test "${CAPACITY_CELL_ID}" != none; then topology_args=( @@ -207,6 +222,7 @@ jobs: "${floor_args[@]}" \ --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ "${identity_args[@]}" \ + "${shape_args[@]}" \ --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ --rehome-audience "${REHOME_AUDIENCE}" \ --rehome-control-origin https://relay-staging.onorca.dev \ diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index 55ef3f894ad..0266bd53add 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -142,6 +142,7 @@ "google_logging_metric.relay_cell_reserve_dead_man_tripped", "google_logging_metric.relay_control_flags_unreadable", "google_logging_metric.relay_incident", + "google_logging_metric.relay_event_loop_stall", "google_logging_metric.relay_long_cell_inventory_hold", "google_logging_metric.relay_reserve_placement_off_with_reserve_cells", "google_logging_metric.relay_snapshot", @@ -167,6 +168,7 @@ "google_monitoring_alert_policy.relay_far_cell_accept_latency", "google_monitoring_alert_policy.relay_gce_connection_headroom", "google_monitoring_alert_policy.relay_lock_timeout_burst", + "google_monitoring_alert_policy.relay_event_loop_stall", "google_monitoring_alert_policy.relay_long_cell_inventory_hold", "google_monitoring_alert_policy.relay_postgres_retry_exhausted", "google_monitoring_alert_policy.relay_process_fatal", @@ -216,6 +218,7 @@ "google_secret_manager_secret.push_provider", "google_secret_manager_secret.relay_assignment_signing_key", "google_secret_manager_secret.relay_database_url", + "google_secret_manager_secret.relay_director_database_url", "google_secret_manager_secret.relay_regional_placement_enabled", "google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor", "google_secret_manager_secret_iam_member.push_provider_runtime_accessor", @@ -223,6 +226,7 @@ "google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor", "google_secret_manager_secret_iam_member.relay_database_url_accessor", "google_secret_manager_secret_iam_member.relay_database_url_director_accessor", + "google_secret_manager_secret_iam_member.relay_director_database_url_accessor", "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor", "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder", "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer", @@ -231,6 +235,7 @@ "google_secret_manager_secret_version.push_dedicated_database_url", "google_secret_manager_secret_version.relay_assignment_signing_key", "google_secret_manager_secret_version.relay_database_url", + "google_secret_manager_secret_version.relay_director_database_url", "google_secret_manager_secret_version.relay_regional_placement_enabled", "google_service_account.github_fence", "google_service_account.github_monitor", diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs index d93b2617e86..08a18475cab 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs @@ -377,6 +377,28 @@ export function parseArguments(argv) { if (!['true', 'false'].includes(values['bootstrap-runtime-identity'] ?? 'false')) { throw new Error('--bootstrap-runtime-identity must be true or false') } + if ( + values['shape-baseline-revision'] !== undefined && + !/^[a-z][a-z0-9-]{0,62}$/.test(values['shape-baseline-revision']) + ) { + throw new Error('--shape-baseline-revision must be a Cloud Run revision name') + } + if (values['shape-baseline-revision'] !== undefined) { + if (values.role !== 'director') { + throw new Error('--shape-baseline-revision requires --role director') + } + // A shape adoption moves only what the reviewed apply changed: no identity, topology or + // pruning rides along, whichever caller builds the arguments. + if (values['bootstrap-runtime-identity'] === 'true') { + throw new Error('--shape-baseline-revision cannot combine with a runtime identity bootstrap') + } + if (values['capacity-cell-id'] !== undefined || values['director-cells-json'] !== undefined) { + throw new Error('--shape-baseline-revision cannot combine with a director topology change') + } + if (values['prune-revisions'] === 'true') { + throw new Error('--shape-baseline-revision cannot combine with revision pruning') + } + } if (values.role === 'director') { directorDeploymentEnvironment(values) projectServiceAccount(values, 'runtime-service-account') @@ -896,6 +918,31 @@ function assertDirectorRevisionIdentity(revision, config) { } } +// The reviewed shape a deploy must preserve. Normally the serving revision's; after a Terraform +// apply changes the template (a sidecar, a volume, a secret reference), the apply's own 0% +// revision, named by the operator, is the reviewed shape instead. +export function directorShapeBaseline(config, service, servingRevision, describe) { + const named = config['shape-baseline-revision'] + if (named === undefined) return servingRevision + if (named === activeRevision(service)) { + throw new Error('shape baseline revision is already serving; deploy without it') + } + if (named !== service.status?.latestCreatedRevisionName) { + throw new Error('shape baseline revision is not the latest created revision') + } + const servingImage = servingRevision.spec?.containers?.[0]?.image + // A shape change and an image change never ride together: each is reviewed on its own. + if (config.image?.split('@').at(-1) !== servingImage?.split('@').at(-1)) { + throw new Error('a shape baseline deploy must keep the serving image') + } + const baseline = describe(config, named) + // Terraform ignores the image, so its revision runs what is serving; anything else was not an apply. + if (baseline.spec?.containers?.[0]?.image !== servingImage) { + throw new Error('shape baseline revision does not run the serving image') + } + return baseline +} + export async function deployDirector(config, tag, overrides = {}) { const operations = { deployCandidate, @@ -916,6 +963,12 @@ export async function deployDirector(config, tag, overrides = {}) { // restating it here; public admission is per-instance, so losing it shrinks fleet capacity. const initialService = operations.describeService(config) const servingRevision = operations.describeRevision(config, activeRevision(initialService)) + const shapeBaseline = directorShapeBaseline( + config, + initialService, + servingRevision, + operations.describeRevision + ) const bootstrapRuntimeIdentity = config['bootstrap-runtime-identity'] === 'true' if (bootstrapRuntimeIdentity) { if ( @@ -1023,7 +1076,7 @@ export async function deployDirector(config, tag, overrides = {}) { const rollbackRevision = operations.describeRevision(config, rollback.revision) assertDirectorRevisionIdentity(rollbackRevision, config) assertPreservedRevisionShape( - servingRevision, + shapeBaseline, rollbackRevision, mutableEnvironment, bootstrapRuntimeIdentity @@ -1058,7 +1111,7 @@ export async function deployDirector(config, tag, overrides = {}) { const candidateRevision = operations.describeRevision(config, candidate.revision) assertDirectorRevisionIdentity(candidateRevision, config) assertPreservedRevisionShape( - servingRevision, + shapeBaseline, candidateRevision, mutableEnvironment, bootstrapRuntimeIdentity diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs index 4ab531b58a7..0ba40b6f97b 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs @@ -441,7 +441,9 @@ function directorHarness({ reserveCells = {}, servingEnvironment = {}, placements = {}, - callerEmail + callerEmail, + // The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one. + templateTransport } = {}) { const state = { activeRevision: 'relay-00001-old', @@ -468,7 +470,9 @@ function directorHarness({ } ] ]), - nextRevision: 2 + nextRevision: 2, + latestCreated: 'relay-00001-old', + templateTransport } const removed = [] const healthProtocols = [] @@ -477,6 +481,7 @@ function directorHarness({ describeService: () => ({ status: { url: 'https://relay-hash-uc.a.run.app', + latestCreatedRevisionName: state.latestCreated, traffic: [ { percent: 100, revisionName: state.activeRevision }, ...[...state.tags].map(([tag, revisionName]) => ({ @@ -493,13 +498,15 @@ function directorHarness({ metadata: { annotations: { 'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0), - 'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5) + 'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5), + ...value?.transport?.annotations } }, spec: { serviceAccountName: value?.serviceAccount, containers: [ { + name: value?.transport?.name, image: value?.image, env: [ ...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })), @@ -508,7 +515,8 @@ function directorHarness({ valueSource: { secretKeyRef } })) ] - } + }, + ...value?.transport?.sidecars ?? [] ] } } @@ -529,8 +537,10 @@ function directorHarness({ serviceAccount: config['runtime-service-account'] ?? state.revisions.get(state.activeRevision)?.serviceAccount, - image: image ?? state.revisions.get(state.activeRevision)?.image + image: image ?? state.revisions.get(state.activeRevision)?.image, + transport: state.templateTransport }) + state.latestCreated = revision state.tags.set(tag, revision) if (deployFailure) throw deployFailure }, @@ -715,6 +725,105 @@ test('director deploy refuses to move traffic onto a candidate that lost the flo assert.equal(harness.state.activeRevision, 'relay-00001-old') }) +const SERVING_IMAGE = `relay@sha256:${'f'.repeat(64)}` + +// What a Terraform apply of relay_director_cloud_sql_proxy_sidecar puts in the template. +const SIDECAR_TRANSPORT = { + name: 'relay', + annotations: { 'run.googleapis.com/container-dependencies': '{"relay":["cloud-sql-proxy"]}' }, + sidecars: [{ name: 'cloud-sql-proxy', image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:1' }] +} + +// The apply leaves its revision at 0%, as the director service declares no traffic block. +function applySidecarTemplate(harness) { + const serving = harness.state.revisions.get(harness.state.activeRevision) + harness.state.templateTransport = SIDECAR_TRANSPORT + harness.state.revisions.set('relay-00009-terraform', { ...serving, transport: SIDECAR_TRANSPORT }) + harness.state.latestCreated = 'relay-00009-terraform' +} + +test('a deploy after a sidecar apply refuses the new shape unless the apply revision is named', async () => { + const harness = directorHarness() + applySidecarTemplate(harness) + await assert.rejects( + deployDirector({}, 'candidate-new', harness.operations), + /unrelated revision shape/ + ) + assert.equal(harness.state.activeRevision, 'relay-00001-old') +}) + +test('a deploy adopts the shape of the named Terraform apply revision', async () => { + const harness = directorHarness() + applySidecarTemplate(harness) + await deployDirector( + { image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, + 'candidate-new', + harness.operations + ) + const active = harness.state.revisions.get(harness.state.activeRevision) + assert.equal(active.transport, SIDECAR_TRANSPORT) + assert.equal( + harness.state.revisions.get(harness.state.tags.get('selector-rollback')).transport, + SIDECAR_TRANSPORT + ) +}) + +test('the named shape baseline must be the unserved latest revision on the serving image', async () => { + const stale = directorHarness() + applySidecarTemplate(stale) + await assert.rejects( + deployDirector({ 'shape-baseline-revision': 'relay-00000-stale' }, 'candidate-new', stale.operations), + /not the latest created revision/ + ) + const serving = directorHarness() + await assert.rejects( + deployDirector({ 'shape-baseline-revision': 'relay-00001-old' }, 'candidate-new', serving.operations), + /already serving/ + ) + const otherImage = directorHarness() + applySidecarTemplate(otherImage) + otherImage.state.revisions.get('relay-00009-terraform').image = `relay@sha256:${'e'.repeat(64)}` + await assert.rejects( + deployDirector({ image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, 'candidate-new', otherImage.operations), + /does not run the serving image/ + ) + const newImage = directorHarness() + applySidecarTemplate(newImage) + await assert.rejects( + deployDirector( + { image: `relay@sha256:${'e'.repeat(64)}`, 'shape-baseline-revision': 'relay-00009-terraform' }, + 'candidate-new', + newImage.operations + ), + /must keep the serving image/ + ) + for (const harness of [stale, serving, otherImage, newImage]) { + assert.equal(harness.state.activeRevision, 'relay-00001-old') + } +}) + +test('--shape-baseline-revision takes only a revision name, and only for a director', () => { + const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', 'i@sha256:' + 'a'.repeat(64), '--release-id', 'x'] + assert.throws( + () => parseArguments([...base, '--role', 'director', '--shape-baseline-revision', 'Bad_Name']), + /must be a Cloud Run revision name/ + ) + assert.throws( + () => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--shape-baseline-revision', 'relay-1']), + /--shape-baseline-revision requires --role director/ + ) + const director = [...base, '--role', 'director', '--shape-baseline-revision', 'relay-1'] + assert.equal(parseArguments(director)['shape-baseline-revision'], 'relay-1') + for (const [extra, refusal] of [ + [['--bootstrap-runtime-identity', 'true'], /cannot combine with a runtime identity bootstrap/], + [['--capacity-cell-id', 'c1', '--director-cells-json', '[]'], /cannot combine with a director topology change/], + [['--director-cells-json', '[]'], /cannot combine with a director topology change/], + [['--prune-revisions', 'true'], /cannot combine with revision pruning/] + ]) { + assert.throws(() => parseArguments([...director, ...extra]), refusal) + } +}) + test('director deploy rejects unrelated revision-shape drift', async () => { const harness = directorHarness() const describeRevision = harness.operations.describeRevision diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars index d26a0ee23f5..942b6a05342 100644 --- a/cloud/infra/terraform/environments/staging.tfvars +++ b/cloud/infra/terraform/environments/staging.tfvars @@ -18,6 +18,14 @@ relay_staging_power_auth_service_name = "orca-cloud-auth-staging" relay_base_url = "https://relay-staging.onorca.dev" relay_min_instances = 0 relay_max_instances = 2 + +# Staging proves these before production turns them on. +relay_director_cloud_sql_proxy_sidecar = true +relay_event_loop_stall_alert = true +# What the serving staging director runs (revision 00171, 2026-10-11). Pinned so the sidecar apply, +# whose revision the next deploy adopts as its shape, does not also drop it to the 80 / 30 s defaults. +relay_director_concurrency = 1000 +relay_director_request_timeout_seconds = 3600 # Staging now exercises the production-shaped GCE data plane exclusively. relay_cells = {} # Keep the stable director mapping Terraform-owned while removing cell mappings. diff --git a/cloud/infra/terraform/relay-database.tf b/cloud/infra/terraform/relay-database.tf index 5dbbd71aa31..6e3e0f1a527 100644 --- a/cloud/infra/terraform/relay-database.tf +++ b/cloud/infra/terraform/relay-database.tf @@ -39,6 +39,40 @@ resource "google_secret_manager_secret_version" "relay_database_url" { ) } +# The director's URL when it reaches Cloud SQL through its proxy sidecar. Cells keep the socket URL. +resource "google_secret_manager_secret" "relay_director_database_url" { + count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 + + project = var.project_id + secret_id = "orca-cloud-relay-director-database-url" + labels = local.relay_shared_labels + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "relay_director_database_url" { + count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 + + secret = google_secret_manager_secret.relay_director_database_url[0].id + secret_data = format( + "postgresql://%s:%s@127.0.0.1:5432/%s", + google_sql_user.relay.name, + random_password.relay_database.result, + google_sql_database.relay.name + ) +} + +resource "google_secret_manager_secret_iam_member" "relay_director_database_url_accessor" { + count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 + + project = var.project_id + secret_id = google_secret_manager_secret.relay_director_database_url[0].secret_id + role = "roles/secretmanager.secretAccessor" + member = google_service_account.relay_director_runtime.member +} + resource "google_secret_manager_secret_iam_member" "relay_database_url_accessor" { project = var.project_id secret_id = google_secret_manager_secret.relay_database_url.secret_id diff --git a/cloud/infra/terraform/relay-observability.tf b/cloud/infra/terraform/relay-observability.tf index e1f99cef4fc..5609d223a7a 100644 --- a/cloud/infra/terraform/relay-observability.tf +++ b/cloud/infra/terraform/relay-observability.tf @@ -850,6 +850,86 @@ resource "google_monitoring_alert_policy" "relay_cell_process_exit" { depends_on = [google_logging_metric.relay_incident] } +# One count per runtime sample whose interval held a 5 s or longer freeze. A count, not a distribution: +# a percentile of the max would hide a single freeze among healthy samples. Gated until staging +# has proved it (variables.tf). +resource "google_logging_metric" "relay_event_loop_stall" { + count = var.relay_event_loop_stall_alert ? 1 : 0 + + project = var.project_id + name = "orca_relay_event_loop_stalls" + description = "Runtime samples whose interval held an event-loop freeze of at least 5,000 ms." + filter = "${local.relay_runtime_log_filter} AND jsonPayload.eventLoopDelayMsMax>=5000" + label_extractors = { + role = "EXTRACT(jsonPayload.role)" + cell_id = "EXTRACT(jsonPayload.cellId)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + + labels { + key = "role" + value_type = "STRING" + description = "Relay process role." + } + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier, or director." + } + } +} + +resource "google_monitoring_alert_policy" "relay_event_loop_stall" { + count = var.relay_event_loop_stall_alert ? 1 : 0 + + project = var.project_id + display_name = "Orca Relay: event loop stall" + combiner = "OR" + enabled = true + # Not paging until staging has shown its noise floor. + notification_channels = [] + + dynamic "conditions" { + for_each = { + "Cloud Run" = "cloud_run_revision" + "GCE cell" = "gce_instance" + } + + content { + display_name = "Event loop froze at least 5,000 ms (${conditions.key})" + + condition_threshold { + # The metric counts only samples at or over 5,000 ms, so any count is a breach. + filter = "resource.type=\"${conditions.value}\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_event_loop_stall[0].name}\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "60s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["metric.label.\"role\"", "metric.label.\"cell_id\""] + } + + trigger { + count = 1 + } + } + } + } + + documentation { + content = "A relay process froze its event loop for 5 s or more. While frozen it serves nothing, polls nothing and no timeout fires, so assigns hang and reserve cells lose their dead-man feed. On a director, look for Cloud SQL connection failures in the same window: a connect on Cloud Run's built-in /cloudsql socket blocks the loop until the connector's dial ends (staging 2026-10-11, production 2026-09-28). Otherwise look for a long synchronous task." + mime_type = "text/markdown" + } +} + # Pages on cell holds only: director holds of 1-2.5 s recur several times a day with rehoming paused, # and pausing rehome does not stop them, so paging on them would ask on-call to do nothing. resource "google_monitoring_alert_policy" "relay_long_cell_inventory_hold" { diff --git a/cloud/infra/terraform/relay.tf b/cloud/infra/terraform/relay.tf index 217f92cfd47..ae7994c5bfc 100644 --- a/cloud/infra/terraform/relay.tf +++ b/cloud/infra/terraform/relay.tf @@ -147,16 +147,25 @@ resource "google_cloud_run_v2_service" "relay" { max_instance_count = var.relay_max_instances } - volumes { - name = "cloudsql" + # Built-in socket only without the sidecar: its connect() blocks the event loop (variables.tf). + dynamic "volumes" { + for_each = var.relay_director_cloud_sql_proxy_sidecar ? [] : ["cloudsql"] - cloud_sql_instance { - instances = [local.relay_database_connection_name] + content { + name = volumes.value + + cloud_sql_instance { + instances = [local.relay_database_connection_name] + } } } + # Stays first: deploys and readers address the relay as containers[0], and gcloud flags without + # --container land on the ingress container. containers { - image = var.relay_cloud_run_image + name = var.relay_director_cloud_sql_proxy_sidecar ? "relay" : null + image = var.relay_cloud_run_image + depends_on = var.relay_director_cloud_sql_proxy_sidecar ? ["cloud-sql-proxy"] : null env { name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED" @@ -194,9 +203,13 @@ resource "google_cloud_run_v2_service" "relay" { container_port = 8080 } - volume_mounts { - name = "cloudsql" - mount_path = "/cloudsql" + dynamic "volume_mounts" { + for_each = var.relay_director_cloud_sql_proxy_sidecar ? [] : ["cloudsql"] + + content { + name = volume_mounts.value + mount_path = "/cloudsql" + } } env { @@ -204,7 +217,9 @@ resource "google_cloud_run_v2_service" "relay" { value_source { secret_key_ref { - secret = google_secret_manager_secret.relay_database_url.secret_id + secret = (var.relay_director_cloud_sql_proxy_sidecar + ? google_secret_manager_secret.relay_director_database_url[0].secret_id + : google_secret_manager_secret.relay_database_url.secret_id) version = "latest" } } @@ -380,6 +395,75 @@ resource "google_cloud_run_v2_service" "relay" { cpu_idle = false } } + + # Accepts a loopback TCP connection at once and dials Cloud SQL in the background, so the pool's + # 2 s connect timeout holds through a start-up or failover. Authenticates as the director's + # runtime account. The GCE cells run the same proxy, over a Unix socket on a full Linux kernel. + dynamic "containers" { + for_each = var.relay_director_cloud_sql_proxy_sidecar ? ["cloud-sql-proxy"] : [] + + content { + name = containers.value + image = var.relay_gce_cloud_sql_proxy_image + args = [ + "--address=127.0.0.1", + "--port=5432", + "--health-check", + # Cloud Run's probe is not documented to reach loopback; only the ingress port is routed. + "--http-address=0.0.0.0", + "--http-port=9090", + "--structured-logs", + "--exit-zero-on-sigterm", + # Lets in-flight statements finish while the relay drains after SIGTERM. + "--max-sigterm-delay=10s", + local.relay_database_connection_name + ] + + # Go sizes its scheduler to the host's cores, not the 0.25 vCPU quota; one keeps it from + # being throttled in bursts. + env { + name = "GOMAXPROCS" + value = "1" + } + + # Ready once it listens; it does not wait for the database, so the relay boots through + # an outage exactly as it does today. + startup_probe { + period_seconds = 1 + timeout_seconds = 1 + failure_threshold = 60 + + http_get { + path = "/startup" + port = 9090 + } + } + + # /liveness answers without the database; failing it means the proxy itself is wedged. + liveness_probe { + period_seconds = 10 + timeout_seconds = 1 + failure_threshold = 3 + + http_get { + path = "/liveness" + port = 9090 + } + } + + # A quarter vCPU carries the pool's few connections. Cloud Run rejects only a total under + # 1 vCPU with concurrency above 1 (validated against the API with the relay's 1 vCPU and + # concurrency 1000), so the sidecar adds 25% to director compute, not 100%. + resources { + limits = { + cpu = "0.25" + memory = "256Mi" + } + + cpu_idle = false + } + } + } } # Deploys update immutable images; Terraform owns service shape and IAM. @@ -407,7 +491,9 @@ resource "google_cloud_run_v2_service" "relay" { google_secret_manager_secret_iam_member.relay_database_url_director_accessor, google_secret_manager_secret_version.relay_assignment_signing_key, google_secret_manager_secret_version.relay_regional_placement_enabled, - google_secret_manager_secret_version.relay_database_url + google_secret_manager_secret_version.relay_database_url, + google_secret_manager_secret_iam_member.relay_director_database_url_accessor, + google_secret_manager_secret_version.relay_director_database_url ] } diff --git a/cloud/infra/terraform/variables.tf b/cloud/infra/terraform/variables.tf index 6be05489c3f..a9ae1b7d923 100644 --- a/cloud/infra/terraform/variables.tf +++ b/cloud/infra/terraform/variables.tf @@ -480,9 +480,21 @@ variable "relay_cloud_sql_private_ip" { default = false } +variable "relay_director_cloud_sql_proxy_sidecar" { + type = bool + description = "Reach Cloud SQL from the director through a Cloud SQL Auth Proxy sidecar on 127.0.0.1 (the GCE cells run the same proxy beside the relay, over a Unix socket), instead of Cloud Run's built-in /cloudsql socket. A connect() on the built-in socket blocks the director's whole event loop until the connector's upstream dial ends, up to 10 s per attempt while the instance starts or fails over." + default = false +} + +variable "relay_event_loop_stall_alert" { + type = bool + description = "Count runtime samples whose event-loop delay max reached 5 s and alert on any. The p99 metric reads normal through a single long freeze." + default = false +} + variable "relay_gce_cloud_sql_proxy_image" { type = string - description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers." + description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers and the director sidecar." default = "gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:fc224915ef435afeb5b2a9421260a0d31986d5c8b7c7f5783c7f5d5885700cd2" validation {