diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index 5d0d430a0a6..200de64d77a 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -134,6 +134,7 @@ "google_iam_workload_identity_pool_provider.github_relay_asia_topology", "google_iam_workload_identity_pool_provider.github_staging_relay_capacity", "google_iam_workload_identity_pool_provider.github_staging_relay_deploy", + "google_logging_metric.relay_assignment_lease_shadow", "google_logging_metric.relay_cell_fix_level", "google_logging_metric.relay_cell_pool_herd", "google_logging_metric.relay_incident", @@ -141,6 +142,7 @@ "google_logging_metric.relay_snapshot", "google_monitoring_alert_policy.relay_assignment_5xx", "google_monitoring_alert_policy.relay_assignment_edge_429", + "google_monitoring_alert_policy.relay_assignment_lease_bad_signature", "google_monitoring_alert_policy.relay_cell_control_close_burst", "google_monitoring_alert_policy.relay_cell_control_rtt", "google_monitoring_alert_policy.relay_cell_outdated_fix_level", @@ -151,12 +153,14 @@ "google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop", "google_monitoring_alert_policy.relay_cloud_sql_disk", "google_monitoring_alert_policy.relay_custom", + "google_monitoring_alert_policy.relay_database_rejection_fenced", "google_monitoring_alert_policy.relay_director_cell_inventory_hold", "google_monitoring_alert_policy.relay_far_cell_accept_latency", "google_monitoring_alert_policy.relay_gce_connection_headroom", "google_monitoring_alert_policy.relay_lock_timeout_burst", "google_monitoring_alert_policy.relay_long_cell_inventory_hold", "google_monitoring_alert_policy.relay_postgres_retry_exhausted", + "google_monitoring_alert_policy.relay_process_fatal", "google_monitoring_alert_policy.relay_region_hint_skew", "google_monitoring_dashboard.relay_incident", "google_project_iam_custom_role.github_production_relay_capacity_mutation", diff --git a/cloud/dev/scripts/relay-db-fault-alerts.test.mjs b/cloud/dev/scripts/relay-db-fault-alerts.test.mjs new file mode 100644 index 00000000000..4f802b0b843 --- /dev/null +++ b/cloud/dev/scripts/relay-db-fault-alerts.test.mjs @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { fileURLToPath } from 'node:url' + +// Why: these alerts key on event names and field paths the relay writes (the rejection fence and +// the lease shadow). A rename on either side silences the alert without failing anything. + +const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') +const terraform = read('../../infra/terraform/relay-observability.tf') + +const block = (kind, name) => { + const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] + assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) + return body +} + +const incidentMetric = (key) => { + const body = new RegExp(`\\n ${key} = \\{([\\s\\S]*?)\\n \\}`).exec(terraform)?.[1] + assert.ok(body, `relay_incident_metrics.${key} not found`) + return body +} + +test('the fence metrics count the exact events on directors and cells', () => { + for (const [key, event] of [ + ['database_rejection_fenced', 'orca_relay_database_rejection_fenced'], + ['process_fatal', 'orca_relay_process_fatal'] + ]) { + const metric = incidentMetric(key) + assert.ok(metric.includes(`jsonPayload.event=\\"${event}\\"`), key) + assert.ok(metric.includes('resource.type=\\"cloud_run_revision\\"'), key) + assert.ok(metric.includes('resource.type=\\"gce_instance\\"'), key) + } +}) + +test('the fenced alert is one fleet-wide series, and the fatal alert leaves cells to the exit alert', () => { + const fenced = block('google_monitoring_alert_policy', 'relay_database_rejection_fenced') + assert.ok( + fenced.includes( + 'query = "sum(increase(logging_googleapis_com:user_orca_relay_database_rejection_fenced[5m])) > 0"' + ) + ) + const fatal = block('google_monitoring_alert_policy', 'relay_process_fatal') + assert.ok(fatal.includes('resource.type=\\"cloud_run_revision\\" AND metric.type=\\"logging.googleapis.com/user/orca_relay_process_fatal\\"')) + assert.doesNotMatch(fatal, /gce_instance/) + // The cell side of a fatal rejection still pages, once, through the exit alert. + block('google_monitoring_alert_policy', 'relay_cell_process_exit') +}) + +test('the bad-signature alert stays in the console', () => { + const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature') + assert.match(alert, /notification_channels = \[\]/) +}) + +test('the lease shadow metrics extract every class the cell writes', () => { + const classes = [ + 'agree', + 'disagree', + 'absent', + 'expired', + 'badSignature', + 'wrongHost', + 'wrongCell', + 'epochBehind', + 'epochAhead' + ] + for (const field of classes) { + assert.match(terraform, new RegExp(`field = "classes\\.${field}"`), field) + } + assert.match(terraform, /field = "dbReadMs\.p99"/) + assert.match(terraform, /field = "dbRefused"/) + const metric = block('google_logging_metric', 'relay_assignment_lease_shadow') + assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_assignment_lease_shadow\\"')) + const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature') + assert.ok(alert.includes('user_orca_relay_assignment_lease_shadow_bad_signature_sum')) +}) diff --git a/cloud/infra/terraform/relay-observability.tf b/cloud/infra/terraform/relay-observability.tf index 2e69003e353..a048a31667f 100644 --- a/cloud/infra/terraform/relay-observability.tf +++ b/cloud/infra/terraform/relay-observability.tf @@ -61,6 +61,15 @@ locals { # close but stamps it with the stream's start, and a log-based metric counts by that stamp. Of # the 1,046 `internal_error` streams on 2026-10-06 18:34Z, 42 were stamped within 10 minutes of # the drop; the median was 2.4 h earlier, so an LB-log rate never shows the pulse. + # Both lines come from the unhandledRejection fence (relay-database-rejection-fence.ts). + database_rejection_fenced = { + description = "Database faults that reached unhandledRejection and were survived: a caller forgot to await a database promise. Should be zero; each one is a bug to find." + filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_database_rejection_fenced\"" + } + process_fatal = { + description = "Unhandled rejections the fence kept fatal (not a database fault); each is followed by a process exit." + filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_process_fatal\"" + } cell_control_abnormal_closes = { description = "Desktop control sockets a GCE cell saw end with no close frame (1006), counted when they closed. Hundreds on one cell in a minute is the load balancer ending streams." filter = "resource.type=\"gce_instance\" AND logName=\"projects/${var.project_id}/logs/cos_containers\" AND jsonPayload.message:\"[orca-relay] control closed \" AND jsonPayload.message:\" code=1006 \"" @@ -1385,3 +1394,148 @@ resource "google_monitoring_dashboard" "relay_incident" { depends_on = [google_logging_metric.relay_incident, google_logging_metric.relay_snapshot] } + +# ticketCheck=shadow (assignment-lease-shadow.ts): one line per cell per minute with traffic. +# The lease never decides an admission, so these observe only. +locals { + relay_assignment_lease_shadow_metrics = { + agree = { field = "classes.agree", description = "Host hellos whose echoed lease was valid and the database admitted." } + disagree = { field = "classes.disagree", description = "Host hellos whose echoed lease was valid but the database refused; step 5 must explain each before a lease alone admits." } + absent = { field = "classes.absent", description = "Host hellos with no echoed lease (desktops that predate the header)." } + expired = { field = "classes.expired", description = "Host hellos whose echoed lease had expired." } + bad_signature = { field = "classes.badSignature", description = "Host hellos whose echoed lease failed signature, key id, or shape checks." } + wrong_host = { field = "classes.wrongHost", description = "Host hellos whose echoed lease named another user or host." } + wrong_cell = { field = "classes.wrongCell", description = "Host hellos whose echoed lease named another cell; expected during drains." } + epoch_behind = { field = "classes.epochBehind", description = "Host hellos whose echoed lease was older than the hello's assignment epoch." } + epoch_ahead = { field = "classes.epochAhead", description = "Host hellos whose echoed lease was newer than the hello's assignment epoch." } + db_refused = { field = "dbRefused", description = "Host hellos the database refused, whatever the echoed lease said." } + db_read_ms = { field = "dbReadMs.p99", description = "p99 of the hello's assignment database read, the read a lease-admitted hello would skip." } + } +} + +resource "google_logging_metric" "relay_assignment_lease_shadow" { + for_each = local.relay_assignment_lease_shadow_metrics + + project = var.project_id + name = "orca_relay_assignment_lease_shadow_${each.key}" + description = each.value.description + filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_assignment_lease_shadow\"" + value_extractor = "EXTRACT(jsonPayload.${each.value.field})" + label_extractors = { + cell_id = "EXTRACT(jsonPayload.cellId)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "DISTRIBUTION" + unit = each.key == "db_read_ms" ? "ms" : "1" + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier." + } + } + + bucket_options { + exponential_buckets { + num_finite_buckets = 24 + growth_factor = 2 + scale = 1 + } + } +} + +# Directors only: a cell's fatal rejection ends in a container exit, which +# relay_cell_process_exit already pages on, so paging here too would double every cell incident. +# The counter still includes cells, for diagnosis. +resource "google_monitoring_alert_policy" "relay_process_fatal" { + project = var.project_id + display_name = "Orca Relay: fatal unhandled rejection on a director" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "Fatal unhandled rejection (Cloud Run)" + + condition_threshold { + filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_process_fatal\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["resource.label.\"service_name\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A director logged `orca_relay_process_fatal` and its instance exited: an unhandled rejection that is not a database fault (a programming error, a schema error, or a bare socket errno). Read the line's `code` and `message`, then the instance stderr stack. Cells log the same line before exiting; the `cell process exits` alert pages for them." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_incident] +} + +# Summed across every process: one database fault reaches every cell and director at once, and +# per-instance series would open ~30 incidents for one cause. +resource "google_monitoring_alert_policy" "relay_database_rejection_fenced" { + project = var.project_id + display_name = "Orca Relay: database rejection survived without a handler" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "Fenced database rejections, fleet-wide" + + # One series across directors and cells, which a single threshold filter cannot span. + condition_prometheus_query_language { + query = "sum(increase(logging_googleapis_com:user_orca_relay_database_rejection_fenced[5m])) > 0" + duration = "0s" + } + } + + documentation { + content = "A database fault reached `unhandledRejection` and the process survived it (`orca_relay_database_rejection_fenced`). Nothing crashed, but some caller started a database promise without awaiting it. Group the lines by `code`, `message` and instance to find it, and fix it. Before the fence, this was a process exit." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_incident] +} + +# Console only: a single desktop sending a damaged lease can hold this up, so it is a lead to +# check, never a page. A fleet-wide key or key-id mismatch reads as every cell at once. +resource "google_monitoring_alert_policy" "relay_assignment_lease_bad_signature" { + project = var.project_id + display_name = "Orca Relay: assignment lease shadow sees bad signatures" + combiner = "OR" + enabled = true + notification_channels = [] + + conditions { + display_name = "Bad lease signatures on a cell for 15 minutes" + + # The class totals are distributions (a value extractor needs one), so the sum is the count. + condition_prometheus_query_language { + query = "sum by (cell_id) (increase(logging_googleapis_com:user_orca_relay_assignment_lease_shadow_bad_signature_sum{monitored_resource=\"gce_instance\"}[15m])) > 0" + duration = "900s" + } + } + + documentation { + content = "A cell with `ticketCheck: shadow` kept seeing assignment leases that fail verification for 15 minutes. The database still decides every admission, so no desktop is affected. One misbehaving client can cause this on one cell; many cells at once points to a key or key-id mismatch between directors and cells, which would block `ticketCheck: enforce`. Compare the `kid` the directors issue with `k1-` on the cell." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_assignment_lease_shadow] +} diff --git a/cloud/package.json b/cloud/package.json index 63b20f20b74..9ec7b478c91 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -21,7 +21,7 @@ "load:relay:model": "node dev/scripts/run-relay-load-model.mjs", "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", - "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", + "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck" },