feat(relay): log metric and console alert for unreadable cell switch files (evidence code) (#27024)

cloud/dev/fixtures/terraform-root-partition/families.json

@@ -137,6 +137,7 @@
     "google_logging_metric.relay_assignment_lease_shadow",
     "google_logging_metric.relay_cell_fix_level",
     "google_logging_metric.relay_cell_pool_herd",
+    "google_logging_metric.relay_control_flags_unreadable",
     "google_logging_metric.relay_incident",
     "google_logging_metric.relay_long_cell_inventory_hold",
     "google_logging_metric.relay_snapshot",
@@ -152,6 +153,7 @@
     "google_monitoring_alert_policy.relay_cloud_sql_backends",
     "google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
     "google_monitoring_alert_policy.relay_cloud_sql_disk",
+    "google_monitoring_alert_policy.relay_control_flags_unreadable",
     "google_monitoring_alert_policy.relay_custom",
     "google_monitoring_alert_policy.relay_database_rejection_fenced",
     "google_monitoring_alert_policy.relay_director_cell_inventory_hold",

cloud/dev/scripts/relay-control-flag-alerts.test.mjs

@@ -0,0 +1,37 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import test from 'node:test'
+import { fileURLToPath } from 'node:url'
+
+// Why: the metric keys on the event name and fields the switch-file reader writes. A rename on
+// either side silences the alert without failing anything.
+
+const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
+const terraform = read('../../infra/terraform/relay-observability.tf')
+const channel = read('../../apps/relay/src/relay-control-flag-channel.ts')
+const cellFlags = read('../../apps/relay/src/cell-flags.ts')
+
+const block = (kind, name) => {
+  const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
+  assert.ok(body, `${kind}.${name} not found in relay-observability.tf`)
+  return body
+}
+
+test('the metric counts the event and fields the reader writes', () => {
+  const unreadableLine = /event: 'orca_relay_control_flags_unreadable',\s*object: input\.objectName,\s*failure,/
+  assert.match(channel, unreadableLine)
+  assert.match(cellFlags, /return `cells\/\$\{cellId\}\.json`/)
+  const metric = block('google_logging_metric', 'relay_control_flags_unreadable')
+  assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_control_flags_unreadable\\"'))
+  assert.ok(metric.includes('REGEXP_EXTRACT(jsonPayload.object, \\"cells/([^.]+)\\")'))
+  assert.ok(metric.includes('failure = "EXTRACT(jsonPayload.failure)"'))
+})
+
+test('the alert fires on one line per cell and stays in the console', () => {
+  const alert = block('google_monitoring_alert_policy', 'relay_control_flags_unreadable')
+  // The reader logs once per change of cause, so a stuck cell writes one line; a higher bar misses it.
+  assert.match(alert, /threshold_value = 0\n/)
+  assert.match(alert, /notification_channels = \[\]/)
+  assert.ok(alert.includes('metric.type=\\"logging.googleapis.com/user/orca_relay_control_flags_unreadable\\"'))
+  assert.ok(alert.includes('group_by_fields      = ["metric.label.\\"cell_id\\""]'))
+})

cloud/infra/terraform/relay-observability.tf

@@ -1539,3 +1539,71 @@ resource "google_monitoring_alert_policy" "relay_assignment_lease_bad_signature"
 
   depends_on = [google_logging_metric.relay_assignment_lease_shadow]
 }
+
+# relay-control-flag-channel.ts logs one line per change of cause, not per failed poll, and nothing
+# on recovery: a cell stuck unreadable writes exactly one line, so any count is the signal.
+resource "google_logging_metric" "relay_control_flags_unreadable" {
+  project     = var.project_id
+  name        = "orca_relay_control_flags_unreadable"
+  description = "Cells that started failing to read their switch file (cells/<cell>.json in the relay-control bucket). The cell keeps its last applied switches."
+  filter      = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_control_flags_unreadable\""
+  label_extractors = {
+    cell_id = "REGEXP_EXTRACT(jsonPayload.object, \"cells/([^.]+)\")"
+    failure = "EXTRACT(jsonPayload.failure)"
+  }
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+    unit        = "1"
+
+    labels {
+      key         = "cell_id"
+      value_type  = "STRING"
+      description = "Durable relay cell identifier."
+    }
+    labels {
+      key         = "failure"
+      value_type  = "STRING"
+      description = "metadata, not-found, http, network, malformed, or void (the object failed the parser)."
+    }
+  }
+}
+
+# Console only until the step-3/5 roll finishes: a single storage 5xx on one cell also writes a line.
+resource "google_monitoring_alert_policy" "relay_control_flags_unreadable" {
+  project               = var.project_id
+  display_name          = "Orca Relay: cell cannot read its switch file"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = []
+
+  conditions {
+    display_name = "Switch-file read failures on a cell"
+
+    condition_threshold {
+      filter          = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_control_flags_unreadable\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 0
+      duration        = "0s"
+
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields      = ["metric.label.\"cell_id\""]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "A cell logged `orca_relay_control_flags_unreadable`: its read of `cells/<cell>.json` in `<project>-relay-control` failed, and it keeps its last applied switches (defaults, all off, if it never read one). The line is written once per change of cause and nothing marks recovery, so this incident closing does not mean the cell recovered: compare the object's generation with the cell's last `orca_relay_cell_flags_applied` line. `failure` says why: `not-found` (no object for this cell), `metadata` or `http` (token, IAM, or storage), `network`, `malformed` or `void` (the object is unparseable or failed the schema; fix the object). Many cells at once points to the bucket or its IAM. A one-off `http` or `network` that the next poll cleared needs nothing."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_control_flags_unreadable]
+}

cloud/package.json

@@ -21,7 +21,7 @@
     "load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
     "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
     "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
-    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
+    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
     "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
     "typecheck": "pnpm -r typecheck"
   },