diff --git a/cloud/apps/relay-ops/src/environment-config.test.ts b/cloud/apps/relay-ops/src/environment-config.test.ts index 321167d7cc8..f3522a300cd 100644 --- a/cloud/apps/relay-ops/src/environment-config.test.ts +++ b/cloud/apps/relay-ops/src/environment-config.test.ts @@ -64,6 +64,7 @@ function terraformCells(environment: 'production' | 'staging'): Array<{ machineType: string capacityRequests: number databasePoolMax: number + connectionHardCap: number | null }> { const tfvars = readFileSync( fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)), @@ -77,7 +78,10 @@ function terraformCells(environment: 'production' | 'staging'): Array<{ zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]), - databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10) + databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10), + connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))( + /connection_hard_cap\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] + ) })) .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId)) } @@ -97,7 +101,8 @@ describe('relay operations environment config', () => { zone: cell.zone, machineType: cell.machineType, capacityRequests: cell.capacityRequests, - databasePoolMax: cell.databasePoolMax + databasePoolMax: cell.databasePoolMax, + connectionHardCap: cell.connectionHardCap })) ).toEqual(expected) } @@ -122,17 +127,20 @@ describe('relay operations environment config', () => { { cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev', region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, + configuredAdmission: false }, { cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev', region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, + configuredAdmission: false }, { cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev', region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, + configuredAdmission: false } ]) diff --git a/cloud/apps/relay-ops/src/environment-config.ts b/cloud/apps/relay-ops/src/environment-config.ts index da3de7765ed..efd890367c8 100644 --- a/cloud/apps/relay-ops/src/environment-config.ts +++ b/cloud/apps/relay-ops/src/environment-config.ts @@ -14,6 +14,8 @@ export type RelayOpsCellConfig = { machineType: RelayOpsMachineType capacityRequests: number databasePoolMax: number + // The cell's rendered connectionHardCap; null when Terraform leaves it unset. + connectionHardCap: number | null configuredAdmission: boolean } @@ -74,6 +76,9 @@ export function relayOpsCellsFromTerraform(input: { required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`) ), databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10), + connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))( + /\bconnection_hard_cap\s*=\s*(\d+)/.exec(body)?.[1] + ), configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true' } }) diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts index 830c3216580..704f55717fa 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts @@ -595,7 +595,15 @@ describe('incident monitor sources', () => { ) const signals = result.source.signals expect(signals[`cell.${reserveCellId}.admission_state`]).toMatchObject({ value: 2 }) - for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked', 'connection_hard_cap']) { + // Why: 10-19 run 39225932592 froze reserve c7 at 731 connections against the 500 default. + const configuredCap = RELAY_OPS_ENVIRONMENTS.production.cells.find( + (cell) => cell.cellId === reserveCellId + )?.connectionHardCap + expect(configuredCap).toBe(1_000) + expect(signals[`cell.${reserveCellId}.connection_hard_cap`]).toMatchObject({ + value: configuredCap + }) + for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked']) { expect(signals[`cell.${reserveCellId}.${suffix}`]).toBeUndefined() } expect(signals['cell.production-gce-c1.heartbeat_age_ms']).toMatchObject({ value: 1_000 }) diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.ts index 82fc3632c49..f69d57b11bd 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.ts @@ -628,6 +628,10 @@ export async function directorSignals( for (const { cell, status } of statuses) { const prefix = `cell.${cell.cellId}` if (status.reserve) { + // cell-status carries the live cap; unset in Terraform keeps the evaluator's default, as there. + if (cell.connectionHardCap !== null) { + addSignal(signals, `${prefix}.connection_hard_cap`, cell.connectionHardCap, nowAt) + } // Legacy admission needs relay_cells.enabled, which only cell-status reads: left missing. if (selector.generation !== 0) { addSignal( diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts index 9a92bf01d53..a0eadd1b163 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts @@ -669,6 +669,20 @@ describe('incident monitor evaluator', () => { expect.objectContaining({ signal: 'cell.production-gce-c1.queued_bytes' }) ) + // The connections ceiling is never skipped: the collector's configured cap, else the default. + const busy = reserveSample() + busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(731) + expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual( + expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 500 }) + ) + busy.sources['director-admin']!.signals['cell.production-gce-c1.connection_hard_cap'] = + signal(1_000) + expect(evaluateIncidentSample(busy, startedAt).status).toBe('green') + busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(1_000) + expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual( + expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 999 }) + ) + const unready = reserveSample() unready.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0) expect(evaluateIncidentSample(unready, startedAt).failures).toContainEqual(