fix(relay-ops): judge a reserve cell's connections against its configured hard cap (#30668)

A reserve cell's cell-status read is refused, so it carried no
connection_hard_cap and the evaluator fell back to the 500 default:
run 39225932592 froze on reserve c7 at 731 connections under its
1000 cap. Read connection_hard_cap from the durable Terraform cells
relay-ops already parses and emit it for reserve cells.

cloud/apps/relay-ops/src/environment-config.test.ts

@@ -64,6 +64,7 @@ function terraformCells(environment: 'production' | 'staging'): Array<{
   machineType: string
   capacityRequests: number
   databasePoolMax: number
+  connectionHardCap: number | null
 }> {
   const tfvars = readFileSync(
     fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)),
@@ -77,7 +78,10 @@ function terraformCells(environment: 'production' | 'staging'): Array<{
       zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
       machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
       capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]),
-      databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10)
+      databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10),
+      connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))(
+        /connection_hard_cap\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]
+      )
     }))
     .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId))
 }
@@ -97,7 +101,8 @@ describe('relay operations environment config', () => {
           zone: cell.zone,
           machineType: cell.machineType,
           capacityRequests: cell.capacityRequests,
-          databasePoolMax: cell.databasePoolMax
+          databasePoolMax: cell.databasePoolMax,
+          connectionHardCap: cell.connectionHardCap
         }))
       ).toEqual(expected)
     }
@@ -122,17 +127,20 @@ describe('relay operations environment config', () => {
       {
         cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev',
         region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4',
-        capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+        capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
+        configuredAdmission: false
       },
       {
         cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev',
         region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4',
-        capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+        capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
+        configuredAdmission: false
       },
       {
         cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev',
         region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4',
-        capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+        capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
+        configuredAdmission: false
       }
     ])
 

cloud/apps/relay-ops/src/environment-config.ts

@@ -14,6 +14,8 @@ export type RelayOpsCellConfig = {
   machineType: RelayOpsMachineType
   capacityRequests: number
   databasePoolMax: number
+  // The cell's rendered connectionHardCap; null when Terraform leaves it unset.
+  connectionHardCap: number | null
   configuredAdmission: boolean
 }
 
@@ -74,6 +76,9 @@ export function relayOpsCellsFromTerraform(input: {
           required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`)
         ),
         databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10),
+        connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))(
+          /\bconnection_hard_cap\s*=\s*(\d+)/.exec(body)?.[1]
+        ),
         configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true'
       }
     })

cloud/apps/relay-ops/src/incident-monitor-sources.test.ts

@@ -595,7 +595,15 @@ describe('incident monitor sources', () => {
     )
     const signals = result.source.signals
     expect(signals[`cell.${reserveCellId}.admission_state`]).toMatchObject({ value: 2 })
-    for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked', 'connection_hard_cap']) {
+    // Why: 10-19 run 39225932592 froze reserve c7 at 731 connections against the 500 default.
+    const configuredCap = RELAY_OPS_ENVIRONMENTS.production.cells.find(
+      (cell) => cell.cellId === reserveCellId
+    )?.connectionHardCap
+    expect(configuredCap).toBe(1_000)
+    expect(signals[`cell.${reserveCellId}.connection_hard_cap`]).toMatchObject({
+      value: configuredCap
+    })
+    for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked']) {
       expect(signals[`cell.${reserveCellId}.${suffix}`]).toBeUndefined()
     }
     expect(signals['cell.production-gce-c1.heartbeat_age_ms']).toMatchObject({ value: 1_000 })

cloud/apps/relay-ops/src/incident-monitor-sources.ts

@@ -628,6 +628,10 @@ export async function directorSignals(
   for (const { cell, status } of statuses) {
     const prefix = `cell.${cell.cellId}`
     if (status.reserve) {
+      // cell-status carries the live cap; unset in Terraform keeps the evaluator's default, as there.
+      if (cell.connectionHardCap !== null) {
+        addSignal(signals, `${prefix}.connection_hard_cap`, cell.connectionHardCap, nowAt)
+      }
       // Legacy admission needs relay_cells.enabled, which only cell-status reads: left missing.
       if (selector.generation !== 0) {
         addSignal(

cloud/apps/relay-ops/src/incident-monitor.test.ts

@@ -669,6 +669,20 @@ describe('incident monitor evaluator', () => {
       expect.objectContaining({ signal: 'cell.production-gce-c1.queued_bytes' })
     )
 
+    // The connections ceiling is never skipped: the collector's configured cap, else the default.
+    const busy = reserveSample()
+    busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(731)
+    expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual(
+      expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 500 })
+    )
+    busy.sources['director-admin']!.signals['cell.production-gce-c1.connection_hard_cap'] =
+      signal(1_000)
+    expect(evaluateIncidentSample(busy, startedAt).status).toBe('green')
+    busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(1_000)
+    expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual(
+      expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 999 })
+    )
+
     const unready = reserveSample()
     unready.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0)
     expect(evaluateIncidentSample(unready, startedAt).failures).toContainEqual(