feat(relay): log metrics and alerts for fenced and fatal rejections and the lease shadow (evidence code) (#26839)

* feat(relay): log metrics and alerts for fenced and fatal rejections and the lease shadow

orca_relay_database_rejection_fenced and orca_relay_process_fatal become
incident counters on directors and cells, each with an alert at the first
event. The lease shadow line gets one distribution metric per class plus
the hello database read p99, and an alert when bad signatures persist on
a cell for 15 minutes.

* fix(relay): one fleet-wide fenced-rejection alert, director-only fatal page, console-only bad-signature alert

A database fault reaches every process at once, so fenced rejections sum
into one series. Cells' fatal rejections already page through the cell
exit alert, so the fatal page covers directors only. The bad-signature
alert can be held up by one client, so it stays in the console.

* fix(relay): fleet-wide fenced-rejection alert as one PromQL sum, like the fix-level alert

* feat(relay): metric for the lease shadow's database-refused count

cloud/dev/fixtures/terraform-root-partition/families.json

@@ -134,6 +134,7 @@
     "google_iam_workload_identity_pool_provider.github_relay_asia_topology",
     "google_iam_workload_identity_pool_provider.github_staging_relay_capacity",
     "google_iam_workload_identity_pool_provider.github_staging_relay_deploy",
+    "google_logging_metric.relay_assignment_lease_shadow",
     "google_logging_metric.relay_cell_fix_level",
     "google_logging_metric.relay_cell_pool_herd",
     "google_logging_metric.relay_incident",
@@ -141,6 +142,7 @@
     "google_logging_metric.relay_snapshot",
     "google_monitoring_alert_policy.relay_assignment_5xx",
     "google_monitoring_alert_policy.relay_assignment_edge_429",
+    "google_monitoring_alert_policy.relay_assignment_lease_bad_signature",
     "google_monitoring_alert_policy.relay_cell_control_close_burst",
     "google_monitoring_alert_policy.relay_cell_control_rtt",
     "google_monitoring_alert_policy.relay_cell_outdated_fix_level",
@@ -151,12 +153,14 @@
     "google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
     "google_monitoring_alert_policy.relay_cloud_sql_disk",
     "google_monitoring_alert_policy.relay_custom",
+    "google_monitoring_alert_policy.relay_database_rejection_fenced",
     "google_monitoring_alert_policy.relay_director_cell_inventory_hold",
     "google_monitoring_alert_policy.relay_far_cell_accept_latency",
     "google_monitoring_alert_policy.relay_gce_connection_headroom",
     "google_monitoring_alert_policy.relay_lock_timeout_burst",
     "google_monitoring_alert_policy.relay_long_cell_inventory_hold",
     "google_monitoring_alert_policy.relay_postgres_retry_exhausted",
+    "google_monitoring_alert_policy.relay_process_fatal",
     "google_monitoring_alert_policy.relay_region_hint_skew",
     "google_monitoring_dashboard.relay_incident",
     "google_project_iam_custom_role.github_production_relay_capacity_mutation",

cloud/dev/scripts/relay-db-fault-alerts.test.mjs

@@ -0,0 +1,76 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import test from 'node:test'
+import { fileURLToPath } from 'node:url'
+
+// Why: these alerts key on event names and field paths the relay writes (the rejection fence and
+// the lease shadow). A rename on either side silences the alert without failing anything.
+
+const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
+const terraform = read('../../infra/terraform/relay-observability.tf')
+
+const block = (kind, name) => {
+  const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
+  assert.ok(body, `${kind}.${name} not found in relay-observability.tf`)
+  return body
+}
+
+const incidentMetric = (key) => {
+  const body = new RegExp(`\\n    ${key} = \\{([\\s\\S]*?)\\n    \\}`).exec(terraform)?.[1]
+  assert.ok(body, `relay_incident_metrics.${key} not found`)
+  return body
+}
+
+test('the fence metrics count the exact events on directors and cells', () => {
+  for (const [key, event] of [
+    ['database_rejection_fenced', 'orca_relay_database_rejection_fenced'],
+    ['process_fatal', 'orca_relay_process_fatal']
+  ]) {
+    const metric = incidentMetric(key)
+    assert.ok(metric.includes(`jsonPayload.event=\\"${event}\\"`), key)
+    assert.ok(metric.includes('resource.type=\\"cloud_run_revision\\"'), key)
+    assert.ok(metric.includes('resource.type=\\"gce_instance\\"'), key)
+  }
+})
+
+test('the fenced alert is one fleet-wide series, and the fatal alert leaves cells to the exit alert', () => {
+  const fenced = block('google_monitoring_alert_policy', 'relay_database_rejection_fenced')
+  assert.ok(
+    fenced.includes(
+      'query    = "sum(increase(logging_googleapis_com:user_orca_relay_database_rejection_fenced[5m])) > 0"'
+    )
+  )
+  const fatal = block('google_monitoring_alert_policy', 'relay_process_fatal')
+  assert.ok(fatal.includes('resource.type=\\"cloud_run_revision\\" AND metric.type=\\"logging.googleapis.com/user/orca_relay_process_fatal\\"'))
+  assert.doesNotMatch(fatal, /gce_instance/)
+  // The cell side of a fatal rejection still pages, once, through the exit alert.
+  block('google_monitoring_alert_policy', 'relay_cell_process_exit')
+})
+
+test('the bad-signature alert stays in the console', () => {
+  const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature')
+  assert.match(alert, /notification_channels = \[\]/)
+})
+
+test('the lease shadow metrics extract every class the cell writes', () => {
+  const classes = [
+    'agree',
+    'disagree',
+    'absent',
+    'expired',
+    'badSignature',
+    'wrongHost',
+    'wrongCell',
+    'epochBehind',
+    'epochAhead'
+  ]
+  for (const field of classes) {
+    assert.match(terraform, new RegExp(`field = "classes\\.${field}"`), field)
+  }
+  assert.match(terraform, /field = "dbReadMs\.p99"/)
+  assert.match(terraform, /field = "dbRefused"/)
+  const metric = block('google_logging_metric', 'relay_assignment_lease_shadow')
+  assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_assignment_lease_shadow\\"'))
+  const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature')
+  assert.ok(alert.includes('user_orca_relay_assignment_lease_shadow_bad_signature_sum'))
+})

cloud/infra/terraform/relay-observability.tf

@@ -61,6 +61,15 @@ locals {
     # close but stamps it with the stream's start, and a log-based metric counts by that stamp. Of
     # the 1,046 `internal_error` streams on 2026-10-06 18:34Z, 42 were stamped within 10 minutes of
     # the drop; the median was 2.4 h earlier, so an LB-log rate never shows the pulse.
+    # Both lines come from the unhandledRejection fence (relay-database-rejection-fence.ts).
+    database_rejection_fenced = {
+      description = "Database faults that reached unhandledRejection and were survived: a caller forgot to await a database promise. Should be zero; each one is a bug to find."
+      filter      = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_database_rejection_fenced\""
+    }
+    process_fatal = {
+      description = "Unhandled rejections the fence kept fatal (not a database fault); each is followed by a process exit."
+      filter      = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_process_fatal\""
+    }
     cell_control_abnormal_closes = {
       description = "Desktop control sockets a GCE cell saw end with no close frame (1006), counted when they closed. Hundreds on one cell in a minute is the load balancer ending streams."
       filter      = "resource.type=\"gce_instance\" AND logName=\"projects/${var.project_id}/logs/cos_containers\" AND jsonPayload.message:\"[orca-relay] control closed \" AND jsonPayload.message:\" code=1006 \""
@@ -1385,3 +1394,148 @@ resource "google_monitoring_dashboard" "relay_incident" {
 
   depends_on = [google_logging_metric.relay_incident, google_logging_metric.relay_snapshot]
 }
+
+# ticketCheck=shadow (assignment-lease-shadow.ts): one line per cell per minute with traffic.
+# The lease never decides an admission, so these observe only.
+locals {
+  relay_assignment_lease_shadow_metrics = {
+    agree         = { field = "classes.agree", description = "Host hellos whose echoed lease was valid and the database admitted." }
+    disagree      = { field = "classes.disagree", description = "Host hellos whose echoed lease was valid but the database refused; step 5 must explain each before a lease alone admits." }
+    absent        = { field = "classes.absent", description = "Host hellos with no echoed lease (desktops that predate the header)." }
+    expired       = { field = "classes.expired", description = "Host hellos whose echoed lease had expired." }
+    bad_signature = { field = "classes.badSignature", description = "Host hellos whose echoed lease failed signature, key id, or shape checks." }
+    wrong_host    = { field = "classes.wrongHost", description = "Host hellos whose echoed lease named another user or host." }
+    wrong_cell    = { field = "classes.wrongCell", description = "Host hellos whose echoed lease named another cell; expected during drains." }
+    epoch_behind  = { field = "classes.epochBehind", description = "Host hellos whose echoed lease was older than the hello's assignment epoch." }
+    epoch_ahead   = { field = "classes.epochAhead", description = "Host hellos whose echoed lease was newer than the hello's assignment epoch." }
+    db_refused    = { field = "dbRefused", description = "Host hellos the database refused, whatever the echoed lease said." }
+    db_read_ms    = { field = "dbReadMs.p99", description = "p99 of the hello's assignment database read, the read a lease-admitted hello would skip." }
+  }
+}
+
+resource "google_logging_metric" "relay_assignment_lease_shadow" {
+  for_each = local.relay_assignment_lease_shadow_metrics
+
+  project         = var.project_id
+  name            = "orca_relay_assignment_lease_shadow_${each.key}"
+  description     = each.value.description
+  filter          = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_assignment_lease_shadow\""
+  value_extractor = "EXTRACT(jsonPayload.${each.value.field})"
+  label_extractors = {
+    cell_id = "EXTRACT(jsonPayload.cellId)"
+  }
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "DISTRIBUTION"
+    unit        = each.key == "db_read_ms" ? "ms" : "1"
+
+    labels {
+      key         = "cell_id"
+      value_type  = "STRING"
+      description = "Durable relay cell identifier."
+    }
+  }
+
+  bucket_options {
+    exponential_buckets {
+      num_finite_buckets = 24
+      growth_factor      = 2
+      scale              = 1
+    }
+  }
+}
+
+# Directors only: a cell's fatal rejection ends in a container exit, which
+# relay_cell_process_exit already pages on, so paging here too would double every cell incident.
+# The counter still includes cells, for diagnosis.
+resource "google_monitoring_alert_policy" "relay_process_fatal" {
+  project               = var.project_id
+  display_name          = "Orca Relay: fatal unhandled rejection on a director"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = var.relay_alert_notification_channels
+
+  conditions {
+    display_name = "Fatal unhandled rejection (Cloud Run)"
+
+    condition_threshold {
+      filter          = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_process_fatal\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 0
+      duration        = "0s"
+
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields      = ["resource.label.\"service_name\""]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "A director logged `orca_relay_process_fatal` and its instance exited: an unhandled rejection that is not a database fault (a programming error, a schema error, or a bare socket errno). Read the line's `code` and `message`, then the instance stderr stack. Cells log the same line before exiting; the `cell process exits` alert pages for them."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_incident]
+}
+
+# Summed across every process: one database fault reaches every cell and director at once, and
+# per-instance series would open ~30 incidents for one cause.
+resource "google_monitoring_alert_policy" "relay_database_rejection_fenced" {
+  project               = var.project_id
+  display_name          = "Orca Relay: database rejection survived without a handler"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = var.relay_alert_notification_channels
+
+  conditions {
+    display_name = "Fenced database rejections, fleet-wide"
+
+    # One series across directors and cells, which a single threshold filter cannot span.
+    condition_prometheus_query_language {
+      query    = "sum(increase(logging_googleapis_com:user_orca_relay_database_rejection_fenced[5m])) > 0"
+      duration = "0s"
+    }
+  }
+
+  documentation {
+    content   = "A database fault reached `unhandledRejection` and the process survived it (`orca_relay_database_rejection_fenced`). Nothing crashed, but some caller started a database promise without awaiting it. Group the lines by `code`, `message` and instance to find it, and fix it. Before the fence, this was a process exit."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_incident]
+}
+
+# Console only: a single desktop sending a damaged lease can hold this up, so it is a lead to
+# check, never a page. A fleet-wide key or key-id mismatch reads as every cell at once.
+resource "google_monitoring_alert_policy" "relay_assignment_lease_bad_signature" {
+  project               = var.project_id
+  display_name          = "Orca Relay: assignment lease shadow sees bad signatures"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = []
+
+  conditions {
+    display_name = "Bad lease signatures on a cell for 15 minutes"
+
+    # The class totals are distributions (a value extractor needs one), so the sum is the count.
+    condition_prometheus_query_language {
+      query    = "sum by (cell_id) (increase(logging_googleapis_com:user_orca_relay_assignment_lease_shadow_bad_signature_sum{monitored_resource=\"gce_instance\"}[15m])) > 0"
+      duration = "900s"
+    }
+  }
+
+  documentation {
+    content   = "A cell with `ticketCheck: shadow` kept seeing assignment leases that fail verification for 15 minutes. The database still decides every admission, so no desktop is affected. One misbehaving client can cause this on one cell; many cells at once points to a key or key-id mismatch between directors and cells, which would block `ticketCheck: enforce`. Compare the `kid` the directors issue with `k1-<sha256(key)[0:8]>` on the cell."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_assignment_lease_shadow]
+}

cloud/package.json

@@ -21,7 +21,7 @@
     "load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
     "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
     "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
-    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
+    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
     "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
     "typecheck": "pnpm -r typecheck"
   },