diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index 89d97154b0a..9067dda7dbe 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -206,6 +206,7 @@ "google_project_iam_member.github_staging_relay_capacity_mutation", "google_project_iam_member.github_staging_relay_capacity_viewer", "google_project_iam_member.github_staging_relay_deploy_compute_viewer", + "google_project_iam_member.github_staging_relay_deploy_logging_viewer", "google_project_iam_member.github_staging_relay_power", "google_project_iam_member.push_runtime_cloudsql_client", "google_project_iam_member.push_runtime_fcm_admin", diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs index 08a18475cab..f9dfe213daf 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs @@ -30,6 +30,14 @@ export const DIRECTOR_ADMISSION_ENVIRONMENT = Object.freeze({ }) const DIRECTOR_STARTUP_PROBE = 'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1' +// Why: Cloud Run routes a new revision's floor instances their first request up to ~110 s after +// they start, so a one-step move lands every assignment on the one pre-started instance. +export const DIRECTOR_STAGED_TRAFFIC_PERCENT = 20 +// Floor instances have started up to ~220 s after the revision and taken ~110 s more to route. +const DIRECTOR_SPREAD_TIMEOUT_MS = 6 * 60 * 1000 +const DIRECTOR_SPREAD_POLL_INTERVAL_MS = 15_000 +// A hung read must not hold the 20/80 split past the deadline. +const DIRECTOR_SPREAD_READ_TIMEOUT_MS = 30_000 export function taggedRevisionOrigin(serviceOrigin, tag) { const url = new URL(serviceOrigin) @@ -461,8 +469,13 @@ export function parseArguments(argv) { return values } -function commandJson(args) { - const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) +function commandJson(args, { timeoutMs } = {}) { + const result = spawnSync('gcloud', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: timeoutMs + }) + if (result.error) throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.error.message}`) if (result.status !== 0) { throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`) } @@ -727,6 +740,59 @@ function cellIdentifier(sourceCellId, releaseId) { return `${sourceCellId.slice(0, available)}--${suffix}` } +// Instances that served a request since `since`, read from Cloud Run's own request log. +function servingInstanceIds(config, revision, since) { + const filter = [ + 'resource.type="cloud_run_revision"', + `resource.labels.service_name="${config.service}"`, + `resource.labels.revision_name="${revision}"`, + `logName="projects/${config.project}/logs/run.googleapis.com%2Frequests"`, + `timestamp>="${since}"` + ].join(' AND ') + const entries = commandJson([ + 'logging', + 'read', + filter, + '--project', + config.project, + '--limit', + '1000', + '--format=json(labels.instanceId)' + ], { timeoutMs: DIRECTOR_SPREAD_READ_TIMEOUT_MS }) + return new Set(entries.map((entry) => entry.labels?.instanceId).filter(Boolean)) +} + +export async function waitForServingInstances( + config, + revision, + required, + since, + { + readInstances = servingInstanceIds, + pollIntervalMs = DIRECTOR_SPREAD_POLL_INTERVAL_MS, + timeoutMs = DIRECTOR_SPREAD_TIMEOUT_MS + } = {} +) { + const deadline = Date.now() + timeoutMs + let instances = 0 + for (;;) { + try { + instances = readInstances(config, revision, since).size + console.warn(JSON.stringify({ event: 'director_candidate_spread', revision, instances, required })) + if (instances >= required) return + } catch (error) { + // A throttled log read is not evidence either way; the deadline still bounds the wait. + console.warn(JSON.stringify({ event: 'director_candidate_spread_unread', revision, error: String(error) })) + } + if (Date.now() >= deadline) { + // Fewer than the floor here can mean Cloud Run packs the partial share onto fewer instances. + console.warn(JSON.stringify({ event: 'director_candidate_spread_timeout', revision, instances, required })) + throw new Error(`candidate ${revision} served from ${instances} of ${required} instances before the deadline`) + } + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)) + } +} + async function waitForHealth(origin, connectionCapacityProtocol) { const response = await fetch(`${origin}/health`, { signal: AbortSignal.timeout(15_000) }) const body = await response.json() @@ -952,6 +1018,7 @@ export async function deployDirector(config, tag, overrides = {}) { deleteRevision, updateTraffic, waitForHealth, + waitForServingInstances, assertRegionalRehomeDisabled, readReserveCells, readDirectorPlacement, @@ -962,7 +1029,8 @@ export async function deployDirector(config, tag, overrides = {}) { // 100% of traffic. Inheriting the serving floor keeps one owner for the number instead of // restating it here; public admission is per-instance, so losing it shrinks fleet capacity. const initialService = operations.describeService(config) - const servingRevision = operations.describeRevision(config, activeRevision(initialService)) + const servingRevisionName = activeRevision(initialService) + const servingRevision = operations.describeRevision(config, servingRevisionName) const shapeBaseline = directorShapeBaseline( config, initialService, @@ -1058,6 +1126,7 @@ export async function deployDirector(config, tag, overrides = {}) { } removeDirectorTrafficTags(config, operations) let deployed + let staged = false let promoted = false try { operations.deployCandidate( @@ -1178,6 +1247,18 @@ export async function deployDirector(config, tag, overrides = {}) { }) ) } + // A floor of one is the pre-started instance itself; nothing else needs to warm. + if (requiredMinimumInstances > 1) { + const since = new Date().toISOString() + staged = true + operations.updateTraffic(config, [`--to-tags=${tag}=${DIRECTOR_STAGED_TRAFFIC_PERCENT}`]) + await operations.waitForServingInstances( + config, + candidate.revision, + requiredMinimumInstances, + since + ) + } operations.updateTraffic(config, [`--to-tags=${tag}=100`]) promoted = true removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG])) @@ -1195,6 +1276,13 @@ export async function deployDirector(config, tag, overrides = {}) { } catch (rollbackError) { recoveryErrors.push(rollbackError) } + } else if (staged) { + // The serving revision still holds most traffic and its warm floor; the cold rollback has neither. + try { + operations.updateTraffic(config, [`--to-revisions=${servingRevisionName}=100`]) + } catch (rollbackError) { + recoveryErrors.push(rollbackError) + } } try { removeDirectorTrafficTags( diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs index 0ba40b6f97b..fac960c12e9 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs @@ -32,7 +32,8 @@ import { taggedRevisionOrigin, taggedTraffic, trafficTags, - waitForEvacuationCapacity + waitForEvacuationCapacity, + waitForServingInstances } from './deploy-relay-blue-green.mjs' // Terraform declares these values; audited blue/green stamps the same values without targeting @@ -443,7 +444,8 @@ function directorHarness({ placements = {}, callerEmail, // The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one. - templateTransport + templateTransport, + spreadFailure } = {}) { const state = { activeRevision: 'relay-00001-old', @@ -476,6 +478,8 @@ function directorHarness({ } const removed = [] const healthProtocols = [] + const trafficMoves = [] + const spreadWaits = [] let pendingCleanupFailure = cleanupFailsBeforeRemoval const operations = { describeService: () => ({ @@ -563,19 +567,25 @@ function directorHarness({ if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision') return } - const promote = args.find((argument) => argument.startsWith('--to-tags=')) - assert.ok(promote) - const tag = promote.slice('--to-tags='.length).split('=')[0] - state.activeRevision = state.tags.get(tag) + const move = args.find((argument) => /^--to-(tags|revisions)=/.test(argument)) + assert.ok(move) + trafficMoves.push(move) + const [target, percent] = move.slice(move.indexOf('=') + 1).split('=') + if (percent !== '100') return + state.activeRevision = move.startsWith('--to-tags=') ? state.tags.get(target) : target }, waitForHealth: async (_origin, connectionCapacityProtocol) => { healthProtocols.push(connectionCapacityProtocol) }, + waitForServingInstances: async (_config, revision, required) => { + spreadWaits.push({ revision, required }) + if (spreadFailure) throw spreadFailure + }, readReserveCells: async (_config, origin) => reserveCells[origin], readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported', adminIdentityToken: () => adminToken(callerEmail) } - return { state, removed, healthProtocols, operations } + return { state, removed, healthProtocols, trafficMoves, spreadWaits, operations } } test('director deploy removes stale and promoted Cloud Run tags', async () => { @@ -701,6 +711,73 @@ test('director candidate inherits the serving warm-instance floor', async () => assert.equal(rollback.minimum, 0) }) +test('a multi-instance floor moves part of the traffic until every floor instance serves', async () => { + const harness = directorHarness({ servingMinimum: 5 }) + await deployDirector({}, 'candidate-new', harness.operations) + assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=20', '--to-tags=candidate-new=100']) + assert.deepEqual(harness.spreadWaits, [{ revision: 'relay-00003-new', required: 5 }]) + assert.equal(harness.state.activeRevision, 'relay-00003-new') +}) + +test('a floor of one moves all traffic at once', async () => { + const harness = directorHarness() + await deployDirector({}, 'candidate-new', harness.operations) + assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=100']) + assert.deepEqual(harness.spreadWaits, []) +}) + +test('a candidate that never spreads returns traffic to the warm serving revision', async () => { + const failure = new Error('candidate served from 1 of 5 instances before the deadline') + const harness = directorHarness({ servingMinimum: 5, spreadFailure: failure }) + await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure) + assert.deepEqual(harness.trafficMoves, [ + '--to-tags=candidate-new=20', + '--to-revisions=relay-00001-old=100' + ]) + assert.equal(harness.state.activeRevision, 'relay-00001-old') + assert.equal(harness.state.tags.size, 0) +}) + +test('the spread wait counts distinct instances that served since the partial move', async () => { + const reads = [new Set(['a']), new Set(['a', 'b']), new Set(['a', 'b', 'c'])] + const seen = [] + await waitForServingInstances({}, 'relay-00003-new', 3, '2026-10-20T05:25:00.000Z', { + readInstances: (_config, revision, since) => { + seen.push([revision, since]) + return reads.shift() + }, + pollIntervalMs: 0, + timeoutMs: 60_000 + }) + assert.equal(seen.length, 3) + assert.deepEqual(seen[0], ['relay-00003-new', '2026-10-20T05:25:00.000Z']) +}) + +test('the spread wait retries a failed log read', async () => { + let reads = 0 + await waitForServingInstances({}, 'relay-00003-new', 2, '2026-10-20T05:25:00.000Z', { + readInstances: () => { + reads += 1 + if (reads === 1) throw new Error('RESOURCE_EXHAUSTED') + return new Set(['a', 'b']) + }, + pollIntervalMs: 0, + timeoutMs: 60_000 + }) + assert.equal(reads, 2) +}) + +test('the spread wait fails at its deadline', async () => { + await assert.rejects( + waitForServingInstances({}, 'relay-00003-new', 5, '2026-10-20T05:25:00.000Z', { + readInstances: () => new Set(['a']), + pollIntervalMs: 0, + timeoutMs: 0 + }), + /served from 1 of 5 instances before the deadline/ + ) +}) + test('director deploy rejects a serving maximum above the checked budget', async () => { const harness = directorHarness({ servingMaximum: 6 }) await assert.rejects( diff --git a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs index 1f4d946557c..66168db1309 100644 --- a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs +++ b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs @@ -150,6 +150,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () => 'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer', 'google_iam_workload_identity_pool_provider.github_staging_relay_deploy', 'google_project_iam_member.github_staging_relay_deploy_compute_viewer', + 'google_project_iam_member.github_staging_relay_deploy_logging_viewer', 'google_service_account.github_staging_relay_deploy', 'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user', 'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user', @@ -189,7 +190,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () => const projectRoles = declaredFamilies() .filter((family) => family.startsWith('google_project_iam_member.')) .map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1]) - assert.deepEqual(projectRoles, ['roles/compute.viewer']) + assert.deepEqual(projectRoles, ['roles/compute.viewer', 'roles/logging.viewer']) assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/) assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/) }) diff --git a/cloud/docs/orca-relay-operations.md b/cloud/docs/orca-relay-operations.md index 5788ca6d359..2e23041dc36 100644 --- a/cloud/docs/orca-relay-operations.md +++ b/cloud/docs/orca-relay-operations.md @@ -241,7 +241,19 @@ Every later change must use the selector CAS. Deploy the selector-compatible director before cutover. Blue/green deployment creates a separate `selector-rollback` revision at minimum instances zero, promotes a second compatible revision, and retains older revisions through the -stabilization window. Cutover removes every older revision and refuses to +stabilization window. With a floor above one, promotion first moves 20% of +traffic and waits (up to six minutes) until every floor instance has served a +request, because Cloud Run routes new floor instances only ~100 s after they +start; a timeout returns all traffic to the previous serving revision. + +If the deploy job is cancelled or loses its runner during that wait, the 20/80 +split stays in place, and both the next deploy and the rehome workflow refuse a +director without one 100% revision. Restore it with +`gcloud run services update-traffic orca-cloud-relay --project onorca-cloud --region us-central1 --to-revisions==100` +(the revision that held 100% before the deploy). A timed-out deploy restores +that itself but removes every tag, including `selector-rollback`, so the rehome +workflow's rollback-digest check refuses (`rollback tag missing`) until the next +successful director deploy. Cutover removes every older revision and refuses to proceed unless only the compatible active and rollback revisions remain. First inspect the current generation and exact membership: diff --git a/cloud/infra/terraform/relay-staging-deploy-iam.tf b/cloud/infra/terraform/relay-staging-deploy-iam.tf index a45652f2f68..74ee413f109 100644 --- a/cloud/infra/terraform/relay-staging-deploy-iam.tf +++ b/cloud/infra/terraform/relay-staging-deploy-iam.tf @@ -170,3 +170,13 @@ resource "google_project_iam_member" "github_staging_relay_deploy_compute_viewer role = "roles/compute.viewer" member = google_service_account.github_staging_relay_deploy[0].member } + +# Deploy Relay Staging step "Deploy director blue/green" with a director floor above one reads +# the candidate's Cloud Run request log to confirm every floor instance serves before the full move. +resource "google_project_iam_member" "github_staging_relay_deploy_logging_viewer" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + project = var.project_id + role = "roles/logging.viewer" + member = google_service_account.github_staging_relay_deploy[0].member +}