From 380a5800fb88264b907fb70c8d4a986f75c243da Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Thu, 17 Sep 2026 01:05:54 -0400 Subject: [PATCH] infra(relay): raise asia-east2 cell pools to 16 and retire four idle cells The three asia-east2 cells sit 176 ms from the Cloud SQL instance in us-central1. Server-side statement time there is 0.2 ms, so a pool slot is held by the round trip, not by the query. At a pool of 10 they measured 94-156 waiters and 2 s waits, and client accepts ran a ~4 s p95 against 222-646 ms in us-central1. Raising those three pools to 16 is the agreed first step; every other cell stays at 10. c4 and c5 join the committed fence set. Both are existing-only capacity the admission selector can never place on again, they carried ~1 connection each on 40-day-old images, and each still holds 10 Postgres connections. The fence set is the prerequisite the fence-source workflow confirms before it drains and attests a cell; it is not itself the resize. c17 and c18 are not fenced here. They are migration-only, and the runbook requires retire-migration-cell to move a migration-only cell to existing-only through a generation-bound selector CAS before it can be fenced. Terraform cannot express that step. The Cloud SQL consumer contract carried two stale numbers: auth at 2 instances when production has run a cap of 20 since 2026-09-04, and a 400-connection ceiling when the live instance reports 500. Both are corrected, and the budget now asserts its headroom in two named gates instead of one aggregate boolean. Those gates fail: auth alone accounts for 200 configured connections and a 215-connection rollout overlap, so the operating maximum is 713 against a usable ceiling of 490. Nothing here caused that, and no pool was lowered to hide it. --- .../production-cloud-sql-app-consumers.json | 14 +++--- ...relay-cloud-sql-connection-budget.test.mjs | 49 ++++++++++++++----- .../terraform/environments/production.tfvars | 21 ++++++-- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/cloud/dev/contracts/production-cloud-sql-app-consumers.json b/cloud/dev/contracts/production-cloud-sql-app-consumers.json index 1cf4ba600dc..29812d1c28d 100644 --- a/cloud/dev/contracts/production-cloud-sql-app-consumers.json +++ b/cloud/dev/contracts/production-cloud-sql-app-consumers.json @@ -1,15 +1,15 @@ { - "comment": "Production Cloud SQL consumers owned by the private orca-cloud application tree (auth and API services). The relay ships without them, so the values the connection budget needs are published here; the private repository binds every field back to its source in its own CI.", - "authInstances": 2, + "comment": "Production Cloud SQL consumers owned by the private orca-cloud application tree (auth and API services). The relay ships without them, so the values the connection budget needs are published here; the private repository binds every field back to its source in its own CI. The mobile push gateway is deliberately absent: it runs on its own orca-cloud-push-db instance and consumes none of this ceiling.", + "authInstances": 20, "authPoolMax": 10, "apiInstances": 10, "apiPoolMax": 5, - "maxConnections": 400, + "maxConnections": 500, "sources": { - "authInstances": "private apps tfvars: auth service max instances", - "authPoolMax": "private auth service: pg.Pool max", - "apiInstances": "private apps tfvars: API service max instances", + "authInstances": "private apps tfvars: auth_max_instances, raised 2 -> 20 by hand on 2026-09-04 after a max of 2 starved desktop token refresh", + "authPoolMax": "private auth service: pg.Pool max on the request pool", + "apiInstances": "private apps tfvars: max_instances for the artifact API service", "apiPoolMax": "private API service: pg.Pool max", - "maxConnections": "Cloud SQL tier default; no max_connections flag is set" + "maxConnections": "measured SHOW max_connections = 500 on the live instance 2026-09-16; no max_connections flag is set, so this is the db-custom-4-15360 tier default and the previous 400 was an incorrect assumption about it" } } diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs index 89d40954f7d..9c1cff924e3 100644 --- a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs +++ b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs @@ -6,26 +6,51 @@ import { readRelayCloudSqlConnectionBudget } from './relay-cloud-sql-connection-budget.mjs' -test('production shared consumers keep allowance and reserve below the ceiling', () => { +const shortfall = (report) => + [ + `cells ${report.consumers.cells}`, + `directors ${report.consumers.directors}`, + `auth ${report.consumers.auth}`, + `api ${report.consumers.api}`, + `= ${report.configuredMaximum} configured`, + `+ ${report.rolloutOverlap.maximum} rollout overlap`, + `+ ${report.maintenanceAdminAllowance} admin allowance`, + `= ${report.operatingMaximum} operating`, + `against ${report.maxConnections} max_connections less a ${report.explicitReserve} reserve` + ].join(', ') + +test('the production budget reads the committed pools and the measured ceiling', () => { const report = readRelayCloudSqlConnectionBudget() - assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 }) - assert.deepEqual(report.asia, { cells: 3, poolMax: 10 }) - assert.equal(report.configuredMaximum, 315) + assert.equal(report.maxConnections, 500) + assert.deepEqual(report.consumers, { cells: 228, directors: 15, auth: 200, api: 50 }) + assert.deepEqual(report.asia, { cells: 3, poolMax: 16 }) + assert.equal(report.configuredMaximum, 493) assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30) assert.equal(report.rolloutOverlap.apiCandidate, 65) - assert.equal(report.rolloutOverlap.authCandidate, 35) + assert.equal(report.rolloutOverlap.authCandidate, 215) assert.equal(report.rolloutOverlap.relayCells, 15) assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15) - assert.equal(report.rolloutOverlap.maximum, 65) + assert.equal(report.rolloutOverlap.maximum, 215) assert.equal(report.maintenanceAdminAllowance, 5) assert.equal(report.explicitReserve, 10) - assert.equal(report.usableCeiling, 390) - assert.equal(report.operatingMaximum, 385) - assert.equal(report.remainingWithinUsableCeiling, 5) - assert.equal(report.budgetedTotal, 395) - assert.equal(report.unallocated, 5) - assert.equal(report.withinBudget, true) + assert.equal(report.usableCeiling, 490) + assert.equal(report.operatingMaximum, 713) +}) + +test('configured pools fit under the ceiling less the stated reserve', () => { + const report = readRelayCloudSqlConnectionBudget() + + assert.ok( + report.configuredMaximum <= report.usableCeiling, + `configured pools exceed the usable ceiling: ${shortfall(report)}` + ) +}) + +test('a serialized rollout still fits under the ceiling less the stated reserve', () => { + const report = readRelayCloudSqlConnectionBudget() + + assert.ok(report.withinBudget, `the operating maximum exceeds the usable ceiling: ${shortfall(report)}`) }) test('fails closed when pool growth consumes the explicit reserve', () => { diff --git a/cloud/infra/terraform/environments/production.tfvars b/cloud/infra/terraform/environments/production.tfvars index 925a171719d..0d6d48c22f6 100644 --- a/cloud/infra/terraform/environments/production.tfvars +++ b/cloud/infra/terraform/environments/production.tfvars @@ -32,7 +32,20 @@ relay_gce_subnetwork_cidr = "10.42.0.0/24" relay_gce_additional_region_subnetwork_cidrs = { "asia-east2" = "10.42.1.0/24" } -relay_gce_fenced_cells = ["production-gce-c1", "production-gce-c2", "production-gce-c3", "production-gce-c6", "production-gce-c11", "production-gce-c12"] +# Fenced cells are retired existing-only capacity: the selector can never place on them again, +# so their MIGs run at zero rather than holding a VM and 10 Postgres connections each. +relay_gce_fenced_cells = [ + "production-gce-c1", + "production-gce-c2", + "production-gce-c3", + # c4/c5 measured ~1 connection each on 40-day-old images (2026-09-16). Fence them only after + # fence-source drains and attests each one; this list is that operation's prerequisite, not its trigger. + "production-gce-c4", + "production-gce-c5", + "production-gce-c6", + "production-gce-c11", + "production-gce-c12" +] # Initial cells stay admission-disabled until production preflight and go-live approval. relay_gce_cells = { "production-gce-c1" = { @@ -350,7 +363,7 @@ relay_gce_cells = { boot_disk_gb = 30 boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" capacity_requests = 6000 - database_pool_max = 10 + database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" initially_enabled = false connection_hard_cap = 3000 @@ -364,7 +377,7 @@ relay_gce_cells = { boot_disk_gb = 30 boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" capacity_requests = 6000 - database_pool_max = 10 + database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" initially_enabled = false connection_hard_cap = 3000 @@ -378,7 +391,7 @@ relay_gce_cells = { boot_disk_gb = 30 boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" capacity_requests = 6000 - database_pool_max = 10 + database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" initially_enabled = false connection_hard_cap = 3000