From 289586947eecb5d7bb809b8387f1ea5a5b816c34 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:40:31 -0400 Subject: [PATCH] feat(relay): alert when a reserve-mode cell trips its dead-man (#27096) Rebased onto main as one commit after its parent squash-merged; kept both sides of the add/add list conflicts with #27024. --- .../terraform-root-partition/families.json | 8 + .../relay-reserve-dead-man-alert.test.mjs | 69 +++++ cloud/infra/terraform/relay-observability.tf | 252 ++++++++++++++++++ cloud/package.json | 2 +- 4 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index 0fa56d63cd8..55ef3f894ad 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -137,9 +137,13 @@ "google_logging_metric.relay_assignment_lease_shadow", "google_logging_metric.relay_cell_fix_level", "google_logging_metric.relay_cell_pool_herd", + "google_logging_metric.relay_cell_reregistration_gave_up", + "google_logging_metric.relay_cell_reregistration_stalled", + "google_logging_metric.relay_cell_reserve_dead_man_tripped", "google_logging_metric.relay_control_flags_unreadable", "google_logging_metric.relay_incident", "google_logging_metric.relay_long_cell_inventory_hold", + "google_logging_metric.relay_reserve_placement_off_with_reserve_cells", "google_logging_metric.relay_snapshot", "google_monitoring_alert_policy.relay_assignment_5xx", "google_monitoring_alert_policy.relay_assignment_edge_429", @@ -148,6 +152,9 @@ "google_monitoring_alert_policy.relay_cell_control_rtt", "google_monitoring_alert_policy.relay_cell_outdated_fix_level", "google_monitoring_alert_policy.relay_cell_pool_herd", + "google_monitoring_alert_policy.relay_cell_reregistration_gave_up", + "google_monitoring_alert_policy.relay_cell_reregistration_stalled", + "google_monitoring_alert_policy.relay_cell_reserve_dead_man_tripped", "google_monitoring_alert_policy.relay_cell_process_exit", "google_monitoring_alert_policy.relay_cloud_nat_port_drops", "google_monitoring_alert_policy.relay_cloud_sql_backends", @@ -164,6 +171,7 @@ "google_monitoring_alert_policy.relay_postgres_retry_exhausted", "google_monitoring_alert_policy.relay_process_fatal", "google_monitoring_alert_policy.relay_region_hint_skew", + "google_monitoring_alert_policy.relay_reserve_placement_off_with_reserve_cells", "google_monitoring_dashboard.relay_incident", "google_project_iam_custom_role.github_production_relay_capacity_mutation", "google_project_iam_custom_role.github_relay_asia_topology_mutation", diff --git a/cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs b/cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs new file mode 100644 index 00000000000..07636c26768 --- /dev/null +++ b/cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { fileURLToPath } from 'node:url' + +// Why: each metric keys on the event name and fields the relay writes. A rename on either side +// silences the alert without failing anything. + +const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') +const terraform = read('../../infra/terraform/relay-observability.tf') +const deadMan = read('../../apps/relay/src/cell-reserve-dead-man.ts') +const registry = read('../../apps/relay/src/host-session-registry.ts') +const server = read('../../apps/relay/src/relay-server.ts') + +const block = (kind, name) => { + const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] + assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) + return body +} + +test('the metric counts the event and cell the dead-man writes', () => { + assert.match(deadMan, /event: 'orca_relay_cell_reserve_dead_man_tripped',\s*cellId: this\.cellId,/) + const metric = block('google_logging_metric', 'relay_cell_reserve_dead_man_tripped') + assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_cell_reserve_dead_man_tripped\\"')) + assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"')) +}) + +test('the alert fires on one trip per cell and notifies', () => { + const alert = block('google_monitoring_alert_policy', 'relay_cell_reserve_dead_man_tripped') + // One line per switch-file generation, so a higher bar misses a trip. + assert.match(alert, /threshold_value = 0\n/) + assert.match(alert, /notification_channels = var\.relay_alert_notification_channels/) + assert.ok(alert.includes('metric.type=\\"logging.googleapis.com/user/orca_relay_cell_reserve_dead_man_tripped\\"')) + assert.ok(alert.includes('group_by_fields = ["metric.label.\\"cell_id\\""]')) +}) + +const notifyingOnAnyLine = (name, event, resource, group) => { + const alert = block('google_monitoring_alert_policy', name) + assert.match(alert, /threshold_value = 0\n/) + assert.match(alert, /notification_channels = var\.relay_alert_notification_channels/) + assert.ok(alert.includes(`resource.type=\\"${resource}\\" AND metric.type=\\"logging.googleapis.com/user/${event}\\"`)) + assert.ok(alert.includes(`group_by_fields = ["metric.label.\\"${group}\\""]`)) +} + +test('the flip-back alerts count the lines the cell writes, per cell', () => { + for (const event of ['orca_relay_cell_reregistration_gave_up', 'orca_relay_cell_reregistration_stalled']) { + // logIdentity() carries cellId; both lines name the reason. + assert.match(registry, new RegExp(`event: '${event}',\\s*\\.\\.\\.this\\.logIdentity\\(\\),[\\s\\S]{0,200}reason:`)) + const name = event.replace('orca_', '') + const metric = block('google_logging_metric', name) + assert.ok(metric.includes(`resource.type=\\"gce_instance\\" AND jsonPayload.event=\\"${event}\\"`)) + assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"')) + assert.ok(metric.includes('reason = "EXTRACT(jsonPayload.reason)"')) + notifyingOnAnyLine(name, event, 'gce_instance', 'cell_id') + } + assert.match(registry, /cellId: this\.config\.cellId/) +}) + +test('the director alert counts its placement-off line on the director service', () => { + const event = 'orca_relay_reserve_placement_off_with_reserve_cells' + assert.match(server, new RegExp(`event: '${event}',\\s*cells,`)) + const metric = block('google_logging_metric', 'relay_reserve_placement_off_with_reserve_cells') + assert.ok( + metric.includes( + `resource.type=\\"cloud_run_revision\\" AND resource.labels.service_name=\\"\${var.relay_cloud_run_service_name}\\" AND jsonPayload.event=\\"${event}\\"` + ) + ) + notifyingOnAnyLine('relay_reserve_placement_off_with_reserve_cells', event, 'cloud_run_revision', 'revision') +}) diff --git a/cloud/infra/terraform/relay-observability.tf b/cloud/infra/terraform/relay-observability.tf index d5bdf31b44a..e1f99cef4fc 100644 --- a/cloud/infra/terraform/relay-observability.tf +++ b/cloud/infra/terraform/relay-observability.tf @@ -1607,3 +1607,255 @@ resource "google_monitoring_alert_policy" "relay_control_flags_unreadable" { depends_on = [google_logging_metric.relay_control_flags_unreadable] } + +# cell-reserve-dead-man.ts logs once per switch-file generation when it trips, so any count is the signal. +resource "google_logging_metric" "relay_cell_reserve_dead_man_tripped" { + project = var.project_id + name = "orca_relay_cell_reserve_dead_man_tripped" + description = "Reserve-mode cells that heard from no placing director for their dead-man window (60-120 s) and went back to admitting through the database." + filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reserve_dead_man_tripped\"" + label_extractors = { + cell_id = "EXTRACT(jsonPayload.cellId)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier." + } + } +} + +resource "google_monitoring_alert_policy" "relay_cell_reserve_dead_man_tripped" { + project = var.project_id + display_name = "Orca Relay: reserve cell lost its placing director" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "A reserve-mode cell tripped its dead-man" + + condition_threshold { + filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reserve_dead_man_tripped\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["metric.label.\"cell_id\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A cell with `admitMode=reserve` in its switch file heard from no placing director (no booking, no feed poll marked `reserver=1`) for its dead-man window, so it went back to database admission and is re-registering its controls. Nobody is disconnected. Postgres still records the cell as reserve, so sweeps keep skipping it while it is heartbeat-live. First find why placement stopped: a director rollback, placement turned off, or every director failing its feed polls. Then, once the cell's `runtime-status` reports `admitModeEffective=db`, run `cloud-operate-relay-production-cell-flags.yml` with `--set admitMode=db` for that cell: it writes db into the switch file and records db in Postgres, so sweeps resume. **This step is required, within hours.** Until it runs, sweeps skip the cell, so its `reserved_requests` and leases only rise, until the database path treats the cell as full. The trip is latched for this switch-file generation: the cell will not return to reserve until a new reserve write, even if a director comes back. Several cells at once points to the directors, not the cells." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_cell_reserve_dead_man_tripped] +} + +# host-session-registry.ts: a flip back closed a control it could not re-register (4409), after 20 refusals. +resource "google_logging_metric" "relay_cell_reregistration_gave_up" { + project = var.project_id + name = "orca_relay_cell_reregistration_gave_up" + description = "Controls a cell closed with 4409 during a flip back to db, because the database kept refusing their lease." + filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reregistration_gave_up\"" + label_extractors = { + cell_id = "EXTRACT(jsonPayload.cellId)" + reason = "EXTRACT(jsonPayload.reason)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier." + } + labels { + key = "reason" + value_type = "STRING" + description = "The refusal: wrong_assignment (the row names another seat) or another database answer." + } + } +} + +resource "google_monitoring_alert_policy" "relay_cell_reregistration_gave_up" { + project = var.project_id + display_name = "Orca Relay: flip back closed a desktop it could not re-register" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "A cell closed a control during its flip back" + + condition_threshold { + filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reregistration_gave_up\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["metric.label.\"cell_id\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "During a flip back to db, a cell asked the database for a control's lease 20 times and was refused each time, so it closed that desktop with 4409 and the desktop re-assigned. `reason` says why: `wrong_assignment` means the host's row names another seat (normally the ledger reconcile repairs that first); anything else is a database answer the lease path does not accept, which needs a code look. A handful on one flip is tolerable; a steady stream means the flip back is disconnecting desktops: stop flipping further cells." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_cell_reregistration_gave_up] +} + +# host-session-registry.ts: one line per 20 consecutive retries of a database that did not answer. +resource "google_logging_metric" "relay_cell_reregistration_stalled" { + project = var.project_id + name = "orca_relay_cell_reregistration_stalled" + description = "Controls whose flip-back lease has failed 20 times in a row because the database did not answer." + filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reregistration_stalled\"" + label_extractors = { + cell_id = "EXTRACT(jsonPayload.cellId)" + reason = "EXTRACT(jsonPayload.reason)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + + labels { + key = "cell_id" + value_type = "STRING" + description = "Durable relay cell identifier." + } + labels { + key = "reason" + value_type = "STRING" + description = "The last database error." + } + } +} + +resource "google_monitoring_alert_policy" "relay_cell_reregistration_stalled" { + project = var.project_id + display_name = "Orca Relay: flip back stalled on the database" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "A cell keeps retrying flip-back leases against an unanswering database" + + condition_threshold { + filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reregistration_stalled\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["metric.label.\"cell_id\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A cell flipping back to db has retried a control's lease 20 times in a row because the database did not answer (a stall, dropped connections, a busy lock). Nobody is disconnected: the cell keeps every control and retries for as long as it is open, and it reports `admitModeEffective=reserve` until it is done, so Postgres keeps reserve and sweeps stay off it. Look at Cloud SQL first. The flag workflow's wait may time out meanwhile; re-run it with `--set admitMode=db` once the database recovers." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_cell_reregistration_stalled] +} + +# relay-server.ts (director): placement is off while the map shows reserve cells; logged once a minute. +resource "google_logging_metric" "relay_reserve_placement_off_with_reserve_cells" { + project = var.project_id + name = "orca_relay_reserve_placement_off_with_reserve_cells" + description = "Directors running with reserve placement off while some cells report reserve mode." + filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND jsonPayload.event=\"orca_relay_reserve_placement_off_with_reserve_cells\"" + label_extractors = { + revision = "EXTRACT(resource.labels.revision_name)" + } + + metric_descriptor { + metric_kind = "DELTA" + value_type = "INT64" + unit = "1" + + labels { + key = "revision" + value_type = "STRING" + description = "Cloud Run revision of the director." + } + } +} + +resource "google_monitoring_alert_policy" "relay_reserve_placement_off_with_reserve_cells" { + project = var.project_id + display_name = "Orca Relay: director placement off while cells are in reserve" + combiner = "OR" + enabled = true + notification_channels = var.relay_alert_notification_channels + + conditions { + display_name = "A director cannot place onto reserve cells" + + condition_threshold { + filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_reserve_placement_off_with_reserve_cells\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "0s" + + aggregations { + alignment_period = "300s" + per_series_aligner = "ALIGN_SUM" + cross_series_reducer = "REDUCE_SUM" + group_by_fields = ["metric.label.\"revision\""] + } + + trigger { + count = 1 + } + } + } + + documentation { + content = "A director is not placing (`placement` in the line: off, dry-run, or configured on but never built; `configured` says what `ORCA_RELAY_RESERVE_PLACEMENT` asked for) while its map shows cells in reserve mode, so it books nothing there and leaves their hosts to the other directors. A director that predates step 5 logs nothing, so a rollback past it shows only as dead-man trips. If every director has placement off, each reserve cell's dead-man trips within 60-120 s (`Orca Relay: reserve cell lost its placing director`). If only some are (a partial rollback, or a manual traffic split), the rest keep the dead-man from tripping, and this alert is the only sign. Put every director on the step-5 revision with placement on, or flip the reserve cells to db (`cloud-operate-relay-production-cell-flags.yml --set admitMode=db`) before rolling the directors back." + mime_type = "text/markdown" + } + + depends_on = [google_logging_metric.relay_reserve_placement_off_with_reserve_cells] +} diff --git a/cloud/package.json b/cloud/package.json index 1b5b928ce4c..94bb3760950 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -21,7 +21,7 @@ "load:relay:model": "node dev/scripts/run-relay-load-model.mjs", "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", - "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", + "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck" },