From 483fa0aca2ce49479fd58dedb6bb01d720d17c5c Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:28:44 -0400 Subject: [PATCH] fix(cloud): compare the Asia topology budget gate against the measured 500-connection default (#22386) The topology workflow's Cloud SQL gate carried a hard-coded 400 for the instance's tier default while the consumer contract records the value measured on the live instance (SHOW max_connections = 500, 2026-09-16, #21163). The gate compares the two and the first production plan run (35815654836) failed silently on that mismatch before Terraform ran. The verified default now lives beside the tier and version it is verified for, as VERIFIED_DEFAULT_MAX_CONNECTIONS, so the contract and the workflow are two independent records of the same measurement and the gate keeps its cross-check. The test pins the new source and forbids a bare literal. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .github/workflows/cloud-deploy-relay-asia-topology.yml | 8 +++++--- cloud/dev/scripts/relay-asia-topology-workflow.test.mjs | 3 +++ cloud/docs/relay-incident-monitor.md | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cloud-deploy-relay-asia-topology.yml b/.github/workflows/cloud-deploy-relay-asia-topology.yml index d22f8c36494..5f594852eb5 100644 --- a/.github/workflows/cloud-deploy-relay-asia-topology.yml +++ b/.github/workflows/cloud-deploy-relay-asia-topology.yml @@ -55,6 +55,8 @@ jobs: CLOUD_SQL_INSTANCE: ${{ inputs.environment == 'production' && 'orca-cloud-auth-db' || 'orca-cloud-staging-auth-db' }} VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360 VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17 + # SHOW max_connections on the live instance, 2026-09-16; no flag is set. + VERIFIED_DEFAULT_MAX_CONNECTIONS: '500' TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }} TF_VARS: ${{ inputs.environment == 'production' && 'environments/production.tfvars' || 'environments/staging.tfvars' }} TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }} @@ -119,13 +121,13 @@ jobs: live_max="${live_flag}" live_source=explicit-flag else - # The verified production database uses Cloud SQL's 400-connection - # default for this exact shape; fail closed if its shape changes. + # No flag: the ceiling is the tier default measured for this exact + # shape; fail closed if the shape changes. test "$(jq -er '.settings.tier' <<< "${instance}")" = \ "${VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER}" test "$(jq -er '.databaseVersion' <<< "${instance}")" = \ "${VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION}" - live_max=400 + live_max="${VERIFIED_DEFAULT_MAX_CONNECTIONS}" live_source=verified-shape-default fi test "${live_max}" = "${checked_max}" diff --git a/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs b/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs index cc34af47a25..d8965f02869 100644 --- a/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs +++ b/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs @@ -90,6 +90,9 @@ test('checks the connection budget and production live ceiling before planning', assert.match(workflow, /select\(\.name == "max_connections"\)/) assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360/) assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17/) + assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS: '500'/) + assert.match(workflow, /live_max="\$\{VERIFIED_DEFAULT_MAX_CONNECTIONS\}"/) + assert.doesNotMatch(workflow, /live_max=\d/) assert.match(workflow, /live_source=verified-shape-default/) assert.match(workflow, /test "\$\(jq -er '\.settings\.tier'/) assert.match(workflow, /test "\$\(jq -er '\.databaseVersion'/) diff --git a/cloud/docs/relay-incident-monitor.md b/cloud/docs/relay-incident-monitor.md index 8804b34fae6..500a2e5a140 100644 --- a/cloud/docs/relay-incident-monitor.md +++ b/cloud/docs/relay-incident-monitor.md @@ -330,7 +330,8 @@ without its segment is a compile error in relay-contract, not a silent gap. latest-sum over 24 healthy hours: mean ~100, 1-minute spikes to 216, with 10 minutes over the old bar of 160 — enough to freeze roughly one in ten 15-minute pre-drain gates on baseline noise. 250 cleared the healthy peaks - measured then and still fired well before the verified 400-connection ceiling; + measured then and still fired well before the 400-connection ceiling assumed at + the time (the live instance measured 500 on 2026-09-16); pool waiters and pool wait latency keep their strict thresholds. Superseded by the 2026-09-17 entry above, which re-measured a grown baseline against the 490-connection budget.