From 9029e84883daeaca7efaed49bfcf98b720f4f710 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:22:21 -0400 Subject: [PATCH] feat(relay): log metric and console alert for unreadable cell switch files (evidence code) (#27024) --- .../terraform-root-partition/families.json | 2 + .../relay-control-flag-alerts.test.mjs | 37 ++++++++++ cloud/infra/terraform/relay-observability.tf | 68 +++++++++++++++++++ cloud/package.json | 2 +- 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 cloud/dev/scripts/relay-control-flag-alerts.test.mjs diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index 200de64d77a..0fa56d63cd8 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -137,6 +137,7 @@ "google_logging_metric.relay_assignment_lease_shadow", "google_logging_metric.relay_cell_fix_level", "google_logging_metric.relay_cell_pool_herd", + "google_logging_metric.relay_control_flags_unreadable", "google_logging_metric.relay_incident", "google_logging_metric.relay_long_cell_inventory_hold", "google_logging_metric.relay_snapshot", @@ -152,6 +153,7 @@ "google_monitoring_alert_policy.relay_cloud_sql_backends", "google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop", "google_monitoring_alert_policy.relay_cloud_sql_disk", + "google_monitoring_alert_policy.relay_control_flags_unreadable", "google_monitoring_alert_policy.relay_custom", "google_monitoring_alert_policy.relay_database_rejection_fenced", "google_monitoring_alert_policy.relay_director_cell_inventory_hold", diff --git a/cloud/dev/scripts/relay-control-flag-alerts.test.mjs b/cloud/dev/scripts/relay-control-flag-alerts.test.mjs new file mode 100644 index 00000000000..95604f29d6e --- /dev/null +++ b/cloud/dev/scripts/relay-control-flag-alerts.test.mjs @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { fileURLToPath } from 'node:url' + +// Why: the metric keys on the event name and fields the switch-file reader writes. A rename on +// either side silences the alert without failing anything. + +const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') +const terraform = read('../../infra/terraform/relay-observability.tf') +const channel = read('../../apps/relay/src/relay-control-flag-channel.ts') +const cellFlags = read('../../apps/relay/src/cell-flags.ts') + +const block = (kind, name) => { + const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] + assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) + return body +} + +test('the metric counts the event and fields the reader writes', () => { + const unreadableLine = /event: 'orca_relay_control_flags_unreadable',\s*object: input\.objectName,\s*failure,/ + assert.match(channel, unreadableLine) + assert.match(cellFlags, /return `cells\/\$\{cellId\}\.json`/) + const metric = block('google_logging_metric', 'relay_control_flags_unreadable') + assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_control_flags_unreadable\\"')) + assert.ok(metric.includes('REGEXP_EXTRACT(jsonPayload.object, \\"cells/([^.]+)\\")')) + assert.ok(metric.includes('failure = "EXTRACT(jsonPayload.failure)"')) +}) + +test('the alert fires on one line per cell and stays in the console', () => { + const alert = block('google_monitoring_alert_policy', 'relay_control_flags_unreadable') + // The reader logs once per change of cause, so a stuck cell writes one line; a higher bar misses it. + assert.match(alert, /threshold_value = 0\n/) + assert.match(alert, /notification_channels = \[\]/) + assert.ok(alert.includes('metric.type=\\"logging.googleapis.com/user/orca_relay_control_flags_unreadable\\"')) + assert.ok(alert.includes('group_by_fields = ["metric.label.\\"cell_id\\""]')) +}) diff --git a/cloud/infra/terraform/relay-observability.tf b/cloud/infra/terraform/relay-observability.tf index a048a31667f..d5bdf31b44a 100644 --- a/cloud/infra/terraform/relay-observability.tf +++ b/cloud/infra/terraform/relay-observability.tf @@ -1539,3 +1539,71 @@ resource "google_monitoring_alert_policy" "relay_assignment_lease_bad_signature" depends_on = [google_logging_metric.relay_assignment_lease_shadow] } + +# relay-control-flag-channel.ts logs one line per change of cause, not per failed poll, and nothing +# on recovery: a cell stuck unreadable writes exactly one line, so any count is the signal. +resource "google_logging_metric" "relay_control_flags_unreadable" { + project = var.project_id + name = "orca_relay_control_flags_unreadable" + description = "Cells that started failing to read their switch file (cells/.json in the relay-control bucket). The cell keeps its last applied switches." + filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_control_flags_unreadable\"" + label_extractors = { + cell_id = "REGEXP_EXTRACT(jsonPayload.object, \"cells/([^.]+)\")" + failure = "EXTRACT(jsonPayload.failure)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier." + } + labels { + key = "failure" + value_type = "STRING" + description = "metadata, not-found, http, network, malformed, or void (the object failed the parser)." + } + } +} + +# Console only until the step-3/5 roll finishes: a single storage 5xx on one cell also writes a line. +resource "google_monitoring_alert_policy" "relay_control_flags_unreadable" { + project = var.project_id + display_name = "Orca Relay: cell cannot read its switch file" + combiner = "OR" + enabled = true + notification_channels = [] + + conditions { + display_name = "Switch-file read failures on a cell" + + condition_threshold { + filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_control_flags_unreadable\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["metric.label.\"cell_id\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A cell logged `orca_relay_control_flags_unreadable`: its read of `cells/.json` in `-relay-control` failed, and it keeps its last applied switches (defaults, all off, if it never read one). The line is written once per change of cause and nothing marks recovery, so this incident closing does not mean the cell recovered: compare the object's generation with the cell's last `orca_relay_cell_flags_applied` line. `failure` says why: `not-found` (no object for this cell), `metadata` or `http` (token, IAM, or storage), `network`, `malformed` or `void` (the object is unparseable or failed the schema; fix the object). Many cells at once points to the bucket or its IAM. A one-off `http` or `network` that the next poll cleared needs nothing." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_control_flags_unreadable] +} diff --git a/cloud/package.json b/cloud/package.json index 9ec7b478c91..1b5b928ce4c 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -21,7 +21,7 @@ "load:relay:model": "node dev/scripts/run-relay-load-model.mjs", "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", - "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", + "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck" },