feat(relay): alert when a reserve-mode cell trips its dead-man (#27096)

Rebased onto main as one commit after its parent squash-merged; kept both sides of the add/add list conflicts with #27024.

cloud/dev/fixtures/terraform-root-partition/families.json

@@ -137,9 +137,13 @@
     "google_logging_metric.relay_assignment_lease_shadow",
     "google_logging_metric.relay_cell_fix_level",
     "google_logging_metric.relay_cell_pool_herd",
+    "google_logging_metric.relay_cell_reregistration_gave_up",
+    "google_logging_metric.relay_cell_reregistration_stalled",
+    "google_logging_metric.relay_cell_reserve_dead_man_tripped",
     "google_logging_metric.relay_control_flags_unreadable",
     "google_logging_metric.relay_incident",
     "google_logging_metric.relay_long_cell_inventory_hold",
+    "google_logging_metric.relay_reserve_placement_off_with_reserve_cells",
     "google_logging_metric.relay_snapshot",
     "google_monitoring_alert_policy.relay_assignment_5xx",
     "google_monitoring_alert_policy.relay_assignment_edge_429",
@@ -148,6 +152,9 @@
     "google_monitoring_alert_policy.relay_cell_control_rtt",
     "google_monitoring_alert_policy.relay_cell_outdated_fix_level",
     "google_monitoring_alert_policy.relay_cell_pool_herd",
+    "google_monitoring_alert_policy.relay_cell_reregistration_gave_up",
+    "google_monitoring_alert_policy.relay_cell_reregistration_stalled",
+    "google_monitoring_alert_policy.relay_cell_reserve_dead_man_tripped",
     "google_monitoring_alert_policy.relay_cell_process_exit",
     "google_monitoring_alert_policy.relay_cloud_nat_port_drops",
     "google_monitoring_alert_policy.relay_cloud_sql_backends",
@@ -164,6 +171,7 @@
     "google_monitoring_alert_policy.relay_postgres_retry_exhausted",
     "google_monitoring_alert_policy.relay_process_fatal",
     "google_monitoring_alert_policy.relay_region_hint_skew",
+    "google_monitoring_alert_policy.relay_reserve_placement_off_with_reserve_cells",
     "google_monitoring_dashboard.relay_incident",
     "google_project_iam_custom_role.github_production_relay_capacity_mutation",
     "google_project_iam_custom_role.github_relay_asia_topology_mutation",

cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs

@@ -0,0 +1,69 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import test from 'node:test'
+import { fileURLToPath } from 'node:url'
+
+// Why: each metric keys on the event name and fields the relay writes. A rename on either side
+// silences the alert without failing anything.
+
+const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
+const terraform = read('../../infra/terraform/relay-observability.tf')
+const deadMan = read('../../apps/relay/src/cell-reserve-dead-man.ts')
+const registry = read('../../apps/relay/src/host-session-registry.ts')
+const server = read('../../apps/relay/src/relay-server.ts')
+
+const block = (kind, name) => {
+  const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
+  assert.ok(body, `${kind}.${name} not found in relay-observability.tf`)
+  return body
+}
+
+test('the metric counts the event and cell the dead-man writes', () => {
+  assert.match(deadMan, /event: 'orca_relay_cell_reserve_dead_man_tripped',\s*cellId: this\.cellId,/)
+  const metric = block('google_logging_metric', 'relay_cell_reserve_dead_man_tripped')
+  assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_cell_reserve_dead_man_tripped\\"'))
+  assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"'))
+})
+
+test('the alert fires on one trip per cell and notifies', () => {
+  const alert = block('google_monitoring_alert_policy', 'relay_cell_reserve_dead_man_tripped')
+  // One line per switch-file generation, so a higher bar misses a trip.
+  assert.match(alert, /threshold_value = 0\n/)
+  assert.match(alert, /notification_channels = var\.relay_alert_notification_channels/)
+  assert.ok(alert.includes('metric.type=\\"logging.googleapis.com/user/orca_relay_cell_reserve_dead_man_tripped\\"'))
+  assert.ok(alert.includes('group_by_fields      = ["metric.label.\\"cell_id\\""]'))
+})
+
+const notifyingOnAnyLine = (name, event, resource, group) => {
+  const alert = block('google_monitoring_alert_policy', name)
+  assert.match(alert, /threshold_value = 0\n/)
+  assert.match(alert, /notification_channels = var\.relay_alert_notification_channels/)
+  assert.ok(alert.includes(`resource.type=\\"${resource}\\" AND metric.type=\\"logging.googleapis.com/user/${event}\\"`))
+  assert.ok(alert.includes(`group_by_fields      = ["metric.label.\\"${group}\\""]`))
+}
+
+test('the flip-back alerts count the lines the cell writes, per cell', () => {
+  for (const event of ['orca_relay_cell_reregistration_gave_up', 'orca_relay_cell_reregistration_stalled']) {
+    // logIdentity() carries cellId; both lines name the reason.
+    assert.match(registry, new RegExp(`event: '${event}',\\s*\\.\\.\\.this\\.logIdentity\\(\\),[\\s\\S]{0,200}reason:`))
+    const name = event.replace('orca_', '')
+    const metric = block('google_logging_metric', name)
+    assert.ok(metric.includes(`resource.type=\\"gce_instance\\" AND jsonPayload.event=\\"${event}\\"`))
+    assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"'))
+    assert.ok(metric.includes('reason  = "EXTRACT(jsonPayload.reason)"'))
+    notifyingOnAnyLine(name, event, 'gce_instance', 'cell_id')
+  }
+  assert.match(registry, /cellId: this\.config\.cellId/)
+})
+
+test('the director alert counts its placement-off line on the director service', () => {
+  const event = 'orca_relay_reserve_placement_off_with_reserve_cells'
+  assert.match(server, new RegExp(`event: '${event}',\\s*cells,`))
+  const metric = block('google_logging_metric', 'relay_reserve_placement_off_with_reserve_cells')
+  assert.ok(
+    metric.includes(
+      `resource.type=\\"cloud_run_revision\\" AND resource.labels.service_name=\\"\${var.relay_cloud_run_service_name}\\" AND jsonPayload.event=\\"${event}\\"`
+    )
+  )
+  notifyingOnAnyLine('relay_reserve_placement_off_with_reserve_cells', event, 'cloud_run_revision', 'revision')
+})

cloud/infra/terraform/relay-observability.tf

@@ -1607,3 +1607,255 @@ resource "google_monitoring_alert_policy" "relay_control_flags_unreadable" {
 
   depends_on = [google_logging_metric.relay_control_flags_unreadable]
 }
+
+# cell-reserve-dead-man.ts logs once per switch-file generation when it trips, so any count is the signal.
+resource "google_logging_metric" "relay_cell_reserve_dead_man_tripped" {
+  project     = var.project_id
+  name        = "orca_relay_cell_reserve_dead_man_tripped"
+  description = "Reserve-mode cells that heard from no placing director for their dead-man window (60-120 s) and went back to admitting through the database."
+  filter      = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reserve_dead_man_tripped\""
+  label_extractors = {
+    cell_id = "EXTRACT(jsonPayload.cellId)"
+  }
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+    unit        = "1"
+
+    labels {
+      key         = "cell_id"
+      value_type  = "STRING"
+      description = "Durable relay cell identifier."
+    }
+  }
+}
+
+resource "google_monitoring_alert_policy" "relay_cell_reserve_dead_man_tripped" {
+  project               = var.project_id
+  display_name          = "Orca Relay: reserve cell lost its placing director"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = var.relay_alert_notification_channels
+
+  conditions {
+    display_name = "A reserve-mode cell tripped its dead-man"
+
+    condition_threshold {
+      filter          = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reserve_dead_man_tripped\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 0
+      duration        = "0s"
+
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields      = ["metric.label.\"cell_id\""]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "A cell with `admitMode=reserve` in its switch file heard from no placing director (no booking, no feed poll marked `reserver=1`) for its dead-man window, so it went back to database admission and is re-registering its controls. Nobody is disconnected. Postgres still records the cell as reserve, so sweeps keep skipping it while it is heartbeat-live. First find why placement stopped: a director rollback, placement turned off, or every director failing its feed polls. Then, once the cell's `runtime-status` reports `admitModeEffective=db`, run `cloud-operate-relay-production-cell-flags.yml` with `--set admitMode=db` for that cell: it writes db into the switch file and records db in Postgres, so sweeps resume. **This step is required, within hours.** Until it runs, sweeps skip the cell, so its `reserved_requests` and leases only rise, until the database path treats the cell as full. The trip is latched for this switch-file generation: the cell will not return to reserve until a new reserve write, even if a director comes back. Several cells at once points to the directors, not the cells."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_cell_reserve_dead_man_tripped]
+}
+
+# host-session-registry.ts: a flip back closed a control it could not re-register (4409), after 20 refusals.
+resource "google_logging_metric" "relay_cell_reregistration_gave_up" {
+  project     = var.project_id
+  name        = "orca_relay_cell_reregistration_gave_up"
+  description = "Controls a cell closed with 4409 during a flip back to db, because the database kept refusing their lease."
+  filter      = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reregistration_gave_up\""
+  label_extractors = {
+    cell_id = "EXTRACT(jsonPayload.cellId)"
+    reason  = "EXTRACT(jsonPayload.reason)"
+  }
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+    unit        = "1"
+
+    labels {
+      key         = "cell_id"
+      value_type  = "STRING"
+      description = "Durable relay cell identifier."
+    }
+    labels {
+      key         = "reason"
+      value_type  = "STRING"
+      description = "The refusal: wrong_assignment (the row names another seat) or another database answer."
+    }
+  }
+}
+
+resource "google_monitoring_alert_policy" "relay_cell_reregistration_gave_up" {
+  project               = var.project_id
+  display_name          = "Orca Relay: flip back closed a desktop it could not re-register"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = var.relay_alert_notification_channels
+
+  conditions {
+    display_name = "A cell closed a control during its flip back"
+
+    condition_threshold {
+      filter          = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reregistration_gave_up\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 0
+      duration        = "0s"
+
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields      = ["metric.label.\"cell_id\""]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "During a flip back to db, a cell asked the database for a control's lease 20 times and was refused each time, so it closed that desktop with 4409 and the desktop re-assigned. `reason` says why: `wrong_assignment` means the host's row names another seat (normally the ledger reconcile repairs that first); anything else is a database answer the lease path does not accept, which needs a code look. A handful on one flip is tolerable; a steady stream means the flip back is disconnecting desktops: stop flipping further cells."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_cell_reregistration_gave_up]
+}
+
+# host-session-registry.ts: one line per 20 consecutive retries of a database that did not answer.
+resource "google_logging_metric" "relay_cell_reregistration_stalled" {
+  project     = var.project_id
+  name        = "orca_relay_cell_reregistration_stalled"
+  description = "Controls whose flip-back lease has failed 20 times in a row because the database did not answer."
+  filter      = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reregistration_stalled\""
+  label_extractors = {
+    cell_id = "EXTRACT(jsonPayload.cellId)"
+    reason  = "EXTRACT(jsonPayload.reason)"
+  }
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+    unit        = "1"
+
+    labels {
+      key         = "cell_id"
+      value_type  = "STRING"
+      description = "Durable relay cell identifier."
+    }
+    labels {
+      key         = "reason"
+      value_type  = "STRING"
+      description = "The last database error."
+    }
+  }
+}
+
+resource "google_monitoring_alert_policy" "relay_cell_reregistration_stalled" {
+  project               = var.project_id
+  display_name          = "Orca Relay: flip back stalled on the database"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = var.relay_alert_notification_channels
+
+  conditions {
+    display_name = "A cell keeps retrying flip-back leases against an unanswering database"
+
+    condition_threshold {
+      filter          = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reregistration_stalled\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 0
+      duration        = "0s"
+
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields      = ["metric.label.\"cell_id\""]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "A cell flipping back to db has retried a control's lease 20 times in a row because the database did not answer (a stall, dropped connections, a busy lock). Nobody is disconnected: the cell keeps every control and retries for as long as it is open, and it reports `admitModeEffective=reserve` until it is done, so Postgres keeps reserve and sweeps stay off it. Look at Cloud SQL first. The flag workflow's wait may time out meanwhile; re-run it with `--set admitMode=db` once the database recovers."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_cell_reregistration_stalled]
+}
+
+# relay-server.ts (director): placement is off while the map shows reserve cells; logged once a minute.
+resource "google_logging_metric" "relay_reserve_placement_off_with_reserve_cells" {
+  project     = var.project_id
+  name        = "orca_relay_reserve_placement_off_with_reserve_cells"
+  description = "Directors running with reserve placement off while some cells report reserve mode."
+  filter      = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND jsonPayload.event=\"orca_relay_reserve_placement_off_with_reserve_cells\""
+  label_extractors = {
+    revision = "EXTRACT(resource.labels.revision_name)"
+  }
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+    unit        = "1"
+
+    labels {
+      key         = "revision"
+      value_type  = "STRING"
+      description = "Cloud Run revision of the director."
+    }
+  }
+}
+
+resource "google_monitoring_alert_policy" "relay_reserve_placement_off_with_reserve_cells" {
+  project               = var.project_id
+  display_name          = "Orca Relay: director placement off while cells are in reserve"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = var.relay_alert_notification_channels
+
+  conditions {
+    display_name = "A director cannot place onto reserve cells"
+
+    condition_threshold {
+      filter          = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_reserve_placement_off_with_reserve_cells\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 0
+      duration        = "0s"
+
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields      = ["metric.label.\"revision\""]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "A director is not placing (`placement` in the line: off, dry-run, or configured on but never built; `configured` says what `ORCA_RELAY_RESERVE_PLACEMENT` asked for) while its map shows cells in reserve mode, so it books nothing there and leaves their hosts to the other directors. A director that predates step 5 logs nothing, so a rollback past it shows only as dead-man trips. If every director has placement off, each reserve cell's dead-man trips within 60-120 s (`Orca Relay: reserve cell lost its placing director`). If only some are (a partial rollback, or a manual traffic split), the rest keep the dead-man from tripping, and this alert is the only sign. Put every director on the step-5 revision with placement on, or flip the reserve cells to db (`cloud-operate-relay-production-cell-flags.yml --set admitMode=db`) before rolling the directors back."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_reserve_placement_off_with_reserve_cells]
+}

cloud/package.json

@@ -21,7 +21,7 @@
     "load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
     "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
     "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
-    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
+    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
     "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
     "typecheck": "pnpm -r typecheck"
   },