import assert from 'node:assert/strict' import { readFileSync } from 'node:fs' import { test } from 'node:test' import { fileURLToPath } from 'node:url' import { activeRevision, correctionCohortPercent, DIRECTOR_CORRECTION_COHORT_ENV, cloudRunTrafficTag, DIRECTOR_ADMISSION_ENVIRONMENT, DIRECTOR_REGIONAL_PLACEMENT_ENV, DIRECTOR_REGIONAL_PLACEMENT_SECRET, DIRECTOR_REHOME_AUDIENCE_ENV, DIRECTOR_REHOME_IDENTITY_ENV, assertRegionalRehomeDisabled, assertReserveCompatibleTrafficMove, assertShadowCellsKeepReserveCells, deployDirector, DIRECTOR_RESERVE_PLACEMENT_ENV, directorDeploymentEnvironment, DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV, directorCellSetAddition, directorStartupProbeArguments, directorTopologyChange, environmentUpdateValue, parseArguments, readDirectorPlacement, readReserveCells, revisionEnvironment, revisionSecretEnvironment, suppliedAdminIdentityToken, taggedRevisionOrigin, taggedTraffic, trafficTags, waitForEvacuationCapacity } from './deploy-relay-blue-green.mjs' // Terraform declares these values; audited blue/green stamps the same values without targeting // the drifted service, so this contract must fail before either side can silently diverge. function terraformDirectorEnvironment(names) { const read = (name) => readFileSync(fileURLToPath(new URL(`../../infra/terraform/${name}`, import.meta.url)), 'utf8') const relay = read('relay.tf') const variables = read('variables.tf') const production = read('environments/production.tfvars') return Object.fromEntries( names.map((name) => { const block = new RegExp(`name\\s*=\\s*"${name}"\\s*\\n\\s*value\\s*=\\s*([^\\n]+)`).exec(relay) assert.ok(block, `${name} is not set by relay.tf`) const variable = /var\.([a-z_]+)/.exec(block[1]) assert.ok(variable, `${name} is not sourced from a Terraform variable`) // An environment override wins over the variable default, as Terraform resolves it. const override = new RegExp(`^${variable[1]}\\s*=\\s*(\\S+)`, 'm').exec(production) const fallback = new RegExp( `variable\\s+"${variable[1]}"[\\s\\S]*?default\\s*=\\s*(\\S+)` ).exec(variables) assert.ok(override || fallback, `${variable[1]} has neither an override nor a default`) return [name, String((override ?? fallback)[1]).replace(/"/g, '')] }) ) } test('director admission environment matches what Terraform deploys', () => { const names = Object.keys(DIRECTOR_ADMISSION_ENVIRONMENT) assert.deepEqual(terraformDirectorEnvironment(names), { ...DIRECTOR_ADMISSION_ENVIRONMENT }) const relay = readFileSync( fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), 'utf8' ) assert.match( relay, /name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED"[\s\S]*?secret\s+= google_secret_manager_secret\.relay_regional_placement_enabled\.secret_id[\s\S]*?version = data\.external\.relay_serving_regional_placement_version\.result\.version/ ) assert.match(relay, /data "external" "relay_serving_regional_placement_version"/) assert.match(relay, /read-relay-serving-regional-placement-version\.mjs/) assert.doesNotMatch(relay, /template\[0\]\.containers\[0\]\.env/) }) test('validates the final stamped-cell admission state', () => { const required = [ '--project', 'project', '--region', 'region', '--service', 'service', '--image', 'image', '--role', 'cell', '--release-id', 'release', '--director-origin', 'https://relay.example.com', '--admin-audience', 'https://relay.example.com/v1/admin/drain' ] assert.equal(parseArguments(required)['final-admission'], undefined) assert.equal( parseArguments([...required, '--final-admission', 'disabled'])['final-admission'], 'disabled' ) assert.throws(() => parseArguments([...required, '--final-admission', 'sometimes'])) assert.equal(parseArguments([...required, '--min-instances', '0'])['min-instances'], '0') assert.throws(() => parseArguments([...required, '--min-instances', '-1'])) assert.throws(() => parseArguments([...required, '--capacity-service-account', 'relay@example.com']) ) }) test('validates optional director capacity configuration', () => { const cells = [ { id: 'staging-gce-c3', url: 'https://c3.relay-staging.onorca.dev', capacityRequests: 4_000, initiallyEnabled: false, region: 'us-central1', connectionHardCap: 600, connectionUnobservedBound: 60 } ] const config = { project: 'onorca-cloud-staging', 'capacity-service-account': 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com', 'director-cells-json': JSON.stringify(cells) } assert.deepEqual(directorDeploymentEnvironment(config), { ...DIRECTOR_ADMISSION_ENVIRONMENT, ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3', ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT: 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com', ORCA_RELAY_CELLS_JSON: JSON.stringify([ { id: cells[0].id, url: cells[0].url, capacityRequests: cells[0].capacityRequests, region: cells[0].region, initiallyEnabled: cells[0].initiallyEnabled, connectionHardCap: cells[0].connectionHardCap, connectionUnobservedBound: cells[0].connectionUnobservedBound } ]) }) assert.throws( () => directorDeploymentEnvironment({ ...config, 'capacity-service-account': 'foreign@other-project.iam.gserviceaccount.com' }), /selected project/ ) assert.throws( () => directorDeploymentEnvironment({ ...config, 'director-cells-json': JSON.stringify([{ ...cells[0], unexpected: true }]) }), /invalid cell/ ) assert.match( environmentUpdateValue(directorDeploymentEnvironment(config)), /^\^~\^ORCA_RELAY_DATABASE_POOL_MAX=/ ) assert.equal(environmentUpdateValue({ FIRST: 'one', SECOND: 'two' }), 'FIRST=one,SECOND=two') assert.deepEqual( directorTopologyChange( JSON.stringify([ { capacityRequests: 4_000, connectionHardCap: 600, connectionUnobservedBound: 60, id: 'staging-gce-c3', initiallyEnabled: false, region: 'us-central1', url: 'https://c3.relay-staging.onorca.dev' } ]), JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]), 'staging-gce-c3' ), { changed: true, value: directorDeploymentEnvironment({ 'director-cells-json': JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]) }).ORCA_RELAY_CELLS_JSON } ) assert.throws( () => directorTopologyChange( JSON.stringify(cells), JSON.stringify([{ ...cells[0], url: 'https://wrong.relay-staging.onorca.dev' }]), 'staging-gce-c3' ), /outside the reviewed capacity pair/ ) }) test('validates exact director runtime and regional rehome identities', () => { const base = [ '--project', 'onorca-cloud', '--region', 'us-central1', '--service', 'orca-cloud-relay', '--image', `relay@sha256:${'a'.repeat(64)}`, '--role', 'director', '--release-id', 'rehome', '--runtime-service-account', 'relay-director@onorca-cloud.iam.gserviceaccount.com', '--rehome-director-service-account', 'relay-director@onorca-cloud.iam.gserviceaccount.com', '--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain', '--expected-rehome-generation', '7', '--rehome-control-origin', 'https://relay.onorca.dev', '--admin-audience', 'https://relay.onorca.dev/v1/admin/drain' ] const config = parseArguments(base) assert.deepEqual(directorDeploymentEnvironment(config), { ...DIRECTOR_ADMISSION_ENVIRONMENT, ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3', ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`, [DIRECTOR_REHOME_IDENTITY_ENV]: 'relay-director@onorca-cloud.iam.gserviceaccount.com', [DIRECTOR_REHOME_AUDIENCE_ENV]: 'https://relay.onorca.dev/v1/admin/host-drain' }) const missingAudience = [...base] missingAudience.splice(missingAudience.indexOf('--rehome-audience'), 2) assert.throws(() => parseArguments(missingAudience), /configured together/) const invalidOrigin = [...base] invalidOrigin[invalidOrigin.indexOf('--rehome-control-origin') + 1] = 'http://relay.onorca.dev' assert.throws( () => parseArguments(invalidOrigin), /HTTPS origin/ ) const mutableImage = [...base] mutableImage[mutableImage.indexOf('--image') + 1] = 'relay:latest' assert.throws(() => parseArguments(mutableImage), /immutable digest/) }) test('requires durable regional rehome control to be disabled at the exact generation', async () => { const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain', 'expected-rehome-generation': '7' } const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' try { const control = await assertRegionalRehomeDisabled( config, 'https://candidate.example.test', async (url, init) => { assert.equal(url, 'https://candidate.example.test/v1/admin/regional-rehome-control') assert.equal(init.headers.authorization, 'Bearer aaa.bbb.ccc') return new Response(JSON.stringify({ v: 1, control: { generation: 7, enabled: false } })) } ) assert.equal(control.generation, 7) await assert.rejects( assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () => new Response(JSON.stringify({ v: 1, control: { generation: 8, enabled: false } })) ), /expected generation/ ) await assert.rejects( assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () => new Response(JSON.stringify({ v: 1, control: { generation: 7, enabled: true } })) ), /durably disabled/ ) } finally { if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment } }) test('rejects literal regional placement changes outside the runtime-setting step', () => { const base = { project: 'onorca-cloud', region: 'us-central1', service: 'orca-cloud-relay', image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`, role: 'director', 'release-id': 'regional-kill-switch' } const args = Object.entries(base).flatMap(([key, value]) => [`--${key}`, value]) assert.doesNotThrow(() => parseArguments(args)) assert.throws( () => parseArguments([...args, '--regional-placement-enabled', 'false']), /audited runtime-setting step/ ) }) test('appends disabled Asia cells without changing the existing director topology', () => { const current = [ { id: 'production-gce-c26', url: 'https://c26.relay.onorca.dev', capacityRequests: 4_000, initiallyEnabled: true, connectionHardCap: 1_000, connectionUnobservedBound: 60 } ] const asia = { id: 'production-gce-c27', url: 'https://c27.relay.onorca.dev', region: 'asia-east2', capacityRequests: 6_000, initiallyEnabled: false, connectionHardCap: 3_000, connectionUnobservedBound: 60 } assert.deepEqual( directorCellSetAddition( JSON.stringify(current), JSON.stringify([asia, { ...current[0], region: 'us-central1' }]) ), { changed: true, value: directorDeploymentEnvironment({ 'director-cells-json': JSON.stringify([asia, { ...current[0], region: 'us-central1' }]) }).ORCA_RELAY_CELLS_JSON } ) const exact = JSON.stringify([{ ...current[0], region: 'us-central1' }, asia]) assert.deepEqual(directorCellSetAddition(exact, exact), { changed: false, value: directorDeploymentEnvironment({ 'director-cells-json': exact }) .ORCA_RELAY_CELLS_JSON }) assert.throws( () => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1', capacityRequests: 6_000 }, asia])), /changes an existing cell/ ) assert.throws( () => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1' }, { ...asia, initiallyEnabled: true }])), /must start disabled/ ) }) test('pins a director startup probe above the bounded reconciliation window', () => { assert.deepEqual(directorStartupProbeArguments('director'), [ '--startup-probe', 'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1' ]) assert.deepEqual(directorStartupProbeArguments('cell'), []) }) test('bounds traffic tags by the Cloud Run service-plus-tag contract', () => { const service = 'orca-cloud-relay-staging-c1' const candidate = cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a') assert.match(candidate, /^candidate-[a-f0-9]{9}$/) assert.equal(service.length + candidate.length, 46) assert.equal(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a')) assert.notEqual(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-2-19cc312a')) assert.throws(() => cloudRunTrafficTag(`${service}-too-long`, 'candidate', 'release')) }) test('derives and validates a Cloud Run tagged revision origin', () => { assert.equal( taggedRevisionOrigin( 'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app', 'candidate-123' ), 'https://candidate-123---orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app' ) assert.throws(() => taggedRevisionOrigin('https://relay-staging.onorca.dev', 'candidate-123')) assert.throws(() => taggedRevisionOrigin( 'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app', '123-invalid' ) ) }) test('requires exactly one active revision and reads queried tag metadata', () => { const service = { status: { traffic: [ { percent: 100, revisionName: 'relay-00001-old' }, { percent: 0, revisionName: 'relay-00002-new', tag: 'candidate-123', url: 'https://candidate-123---relay-hash-uc.a.run.app' } ] } } assert.equal(activeRevision(service), 'relay-00001-old') assert.deepEqual(taggedTraffic(service, 'candidate-123'), { origin: 'https://candidate-123---relay-hash-uc.a.run.app', revision: 'relay-00002-new' }) assert.throws(() => activeRevision({ status: { traffic: [{ percent: 50 }, { percent: 50 }] } }) ) assert.deepEqual(trafficTags(service), ['candidate-123']) }) const adminToken = (email) => `e30.${Buffer.from(JSON.stringify(email ? { email } : {})).toString('base64url')}.sig` function directorHarness({ deployFailure, deleteFailure, cleanupReportsFailure = false, cleanupFailsBeforeRemoval = false, servingMinimum = 1, servingMaximum = 5, // Reproduces gcloud dropping minScale from a newly created revision. dropRequestedMinimum = false, servingServiceAccount, servingImageDigest = `sha256:${'f'.repeat(64)}`, reserveCells = {}, servingEnvironment = {}, placements = {}, callerEmail } = {}) { const state = { activeRevision: 'relay-00001-old', tags: new Map([['candidate-old', 'relay-00000-stale']]), revisions: new Map([ ['relay-00000-stale', { env: { ORCA_RELAY_ROLE: 'director' }, minimum: 1, maximum: 5 }], [ 'relay-00001-old', { env: { ORCA_RELAY_ROLE: 'director', ...servingEnvironment }, secrets: { [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: '1' } }, minimum: servingMinimum, maximum: servingMaximum, serviceAccount: servingServiceAccount, image: `relay@${servingImageDigest}` } ] ]), nextRevision: 2 } const removed = [] const healthProtocols = [] let pendingCleanupFailure = cleanupFailsBeforeRemoval const operations = { describeService: () => ({ status: { url: 'https://relay-hash-uc.a.run.app', traffic: [ { percent: 100, revisionName: state.activeRevision }, ...[...state.tags].map(([tag, revisionName]) => ({ tag, revisionName, url: `https://${tag}---relay-hash-uc.a.run.app` })) ] } }), describeRevision: (_config, revision) => { const value = state.revisions.get(revision) return { metadata: { annotations: { 'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0), 'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5) } }, spec: { serviceAccountName: value?.serviceAccount, containers: [ { image: value?.image, env: [ ...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })), ...Object.entries(value?.secrets ?? {}).map(([name, secretKeyRef]) => ({ name, valueSource: { secretKeyRef } })) ] } ] } } }, deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => { const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new` state.revisions.set(revision, { // gcloud --update-env-vars keeps every variable the deploy does not name. env: { ORCA_RELAY_ROLE: 'director', ...servingEnvironment, ...env }, secrets: { [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: regionalVersion } }, minimum: dropRequestedMinimum ? 0 : Number(minimum ?? 1), maximum, serviceAccount: config['runtime-service-account'] ?? state.revisions.get(state.activeRevision)?.serviceAccount, image: image ?? state.revisions.get(state.activeRevision)?.image }) state.tags.set(tag, revision) if (deployFailure) throw deployFailure }, listRevisions: () => [...state.revisions].map(([name]) => ({ metadata: { name } })), deleteRevision: (_config, revision) => { if (deleteFailure) throw deleteFailure state.revisions.delete(revision) }, updateTraffic: (_config, args) => { const remove = args.find((argument) => argument.startsWith('--remove-tags=')) if (remove) { const tags = remove.slice('--remove-tags='.length).split(',') removed.push(tags) if (pendingCleanupFailure && tags.includes('candidate-new')) { pendingCleanupFailure = false throw new Error('failed to remove promoted candidate tag') } for (const tag of tags) state.tags.delete(tag) if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision') return } const promote = args.find((argument) => argument.startsWith('--to-tags=')) assert.ok(promote) const tag = promote.slice('--to-tags='.length).split('=')[0] state.activeRevision = state.tags.get(tag) }, waitForHealth: async (_origin, connectionCapacityProtocol) => { healthProtocols.push(connectionCapacityProtocol) }, readReserveCells: async (_config, origin) => reserveCells[origin], readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported', adminIdentityToken: () => adminToken(callerEmail) } return { state, removed, healthProtocols, operations } } test('director deploy removes stale and promoted Cloud Run tags', async () => { const harness = directorHarness() const config = { project: 'onorca-cloud-staging', 'capacity-service-account': 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com' } await deployDirector(config, 'candidate-new', harness.operations) assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']]) assert.equal(harness.state.activeRevision, 'relay-00003-new') assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) assert.deepEqual([...harness.state.revisions.keys()], [ 'relay-00000-stale', 'relay-00001-old', 'relay-00002-new', 'relay-00003-new' ]) for (const revision of ['relay-00002-new', 'relay-00003-new']) { assert.deepEqual( Object.fromEntries( Object.entries(harness.state.revisions.get(revision).env).filter(([key]) => key in DIRECTOR_ADMISSION_ENVIRONMENT ) ), DIRECTOR_ADMISSION_ENVIRONMENT ) assert.equal( harness.state.revisions.get(revision).env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT, config['capacity-service-account'] ) } assert.deepEqual(harness.healthProtocols, [undefined, undefined]) }) test('director deploy stamps the durable regional placement secret reference', async () => { const harness = directorHarness() await deployDirector({}, 'candidate-new', harness.operations) for (const revision of ['relay-00002-new', 'relay-00003-new']) { assert.deepEqual( harness.state.revisions.get(revision).secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV], { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: '1' } ) } }) test('bootstraps both rollback and candidate onto the distinct director identity', async () => { const predecessor = 'relay-runtime@onorca-cloud.iam.gserviceaccount.com' const director = 'relay-director@onorca-cloud.iam.gserviceaccount.com' const harness = directorHarness({ servingServiceAccount: predecessor }) const inspected = [] harness.operations.assertRegionalRehomeDisabled = async (_config, origin) => { inspected.push(origin) } await deployDirector({ project: 'onorca-cloud', 'runtime-service-account': director, 'predecessor-runtime-service-account': predecessor, 'predecessor-image-digest': `sha256:${'f'.repeat(64)}`, 'bootstrap-runtime-identity': 'true', 'expected-rehome-generation': '0', 'rehome-control-origin': 'https://relay.onorca.dev' }, 'candidate-new', harness.operations) assert.equal( harness.state.revisions.get(harness.state.activeRevision).serviceAccount, director ) assert.equal( harness.state.revisions.get(harness.state.tags.get('selector-rollback')).serviceAccount, director ) assert.deepEqual(inspected, [ 'https://selector-rollback---relay-hash-uc.a.run.app', 'https://candidate-new---relay-hash-uc.a.run.app' ]) }) test('steady-state director deploy rejects the predecessor identity', async () => { const harness = directorHarness({ servingServiceAccount: 'relay-runtime@onorca-cloud.iam.gserviceaccount.com' }) await assert.rejects(deployDirector({ 'runtime-service-account': 'relay-director@onorca-cloud.iam.gserviceaccount.com' }, 'candidate-new', harness.operations), /unexpected runtime service account/) }) test('director deploy prunes old revisions when requested', async () => { const harness = directorHarness() await deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations) assert.deepEqual([...harness.state.revisions.keys()], [ 'relay-00002-new', 'relay-00003-new' ]) assert.deepEqual(harness.healthProtocols, [2, 2]) }) test('a prune failure preserves the active and rollback traffic pair', async () => { const harness = directorHarness({ deleteFailure: new Error('injected delete failure') }) await assert.rejects( deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations), /injected delete failure/ ) assert.equal(harness.state.activeRevision, 'relay-00003-new') assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) }) test('director deploy removes a candidate tag left by a failed update', async () => { const failure = new Error('candidate failed to become ready') const harness = directorHarness({ deployFailure: failure }) await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure) assert.deepEqual(harness.removed, [['candidate-old'], ['selector-rollback']]) assert.equal(harness.state.tags.size, 0) }) test('director candidate inherits the serving warm-instance floor', async () => { const harness = directorHarness({ servingMinimum: 5 }) await deployDirector({}, 'candidate-new', harness.operations) const serving = harness.state.revisions.get(harness.state.activeRevision) assert.equal(serving.minimum, 5) // The standby rollback revision must stay cold. const rollback = harness.state.revisions.get(harness.state.tags.get('selector-rollback')) assert.equal(rollback.minimum, 0) }) test('director deploy rejects a serving maximum above the checked budget', async () => { const harness = directorHarness({ servingMaximum: 6 }) await assert.rejects( deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations), /holds 6 maximum instances, expected 5/ ) }) test('an explicit --min-instances still overrides the serving floor', async () => { const harness = directorHarness({ servingMinimum: 5 }) await deployDirector({ 'min-instances': '0' }, 'candidate-new', harness.operations) assert.equal(harness.state.revisions.get(harness.state.activeRevision).minimum, 0) }) test('director deploy refuses to move traffic onto a candidate that lost the floor', async () => { const harness = directorHarness({ servingMinimum: 5, dropRequestedMinimum: true }) await assert.rejects( deployDirector({}, 'candidate-new', harness.operations), /candidate holds 0 minimum instances, expected 5/ ) // Traffic never moved, so the original revision still serves. assert.equal(harness.state.activeRevision, 'relay-00001-old') }) test('director deploy rejects unrelated revision-shape drift', async () => { const harness = directorHarness() const describeRevision = harness.operations.describeRevision harness.operations.describeRevision = (config, revision) => { const described = describeRevision(config, revision) described.spec.containerConcurrency = revision === 'relay-00001-old' ? 80 : 1_000 return described } await assert.rejects( deployDirector({}, 'candidate-new', harness.operations), /unrelated revision shape/ ) assert.equal(harness.state.activeRevision, 'relay-00001-old') }) test('director cleanup verifies success when gcloud reports a stale revision failure', async () => { const harness = directorHarness({ cleanupReportsFailure: true }) await deployDirector({}, 'candidate-new', harness.operations) assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']]) assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) }) test('director deploy restores rollback traffic after promoted-tag cleanup fails', async () => { const harness = directorHarness({ cleanupFailsBeforeRemoval: true }) await assert.rejects( deployDirector({}, 'candidate-new', harness.operations), /failed to remove promoted candidate tag/ ) assert.equal( harness.state.activeRevision, harness.state.tags.get('selector-rollback') ) assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) }) test('reads only literal revision environment values', () => { assert.deepEqual( revisionEnvironment({ spec: { containers: [ { env: [ { name: 'ORCA_RELAY_CELL_ID', value: 'staging-c1' }, { name: 'ORCA_RELAY_CELL_CAPACITY', value: '900' }, { name: 'DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database' } } } ] } ] } }), { ORCA_RELAY_CELL_ID: 'staging-c1', ORCA_RELAY_CELL_CAPACITY: '900' } ) }) test('reads only Secret Manager revision environment references', () => { assert.deepEqual( revisionSecretEnvironment({ spec: { containers: [{ env: [ { name: 'LITERAL', value: 'true' }, { name: DIRECTOR_REGIONAL_PLACEMENT_ENV, valueSource: { secretKeyRef: { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: 'latest' } } }, { name: 'GCP_SECRET_SHAPE', valueFrom: { secretKeyRef: { name: 'gcp-secret', key: '2' } } } ] }] } }), { [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: 'latest' }, GCP_SECRET_SHAPE: { secret: 'gcp-secret', version: '2' } } ) }) test('accepts only a bounded JWT-shaped supplied admin identity token', () => { assert.equal(suppliedAdminIdentityToken({}), null) assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc') assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' })) assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' })) assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` }) ) }) test('waits for authenticated target readiness without hiding other capacity errors', async () => { let attempts = 0 const capacity = await waitForEvacuationCapacity( async () => { attempts += 1 if (attempts < 3) throw new Error('/v1/admin/evacuation-capacity failed: target_cell_unavailable') return { requiredTargetUnits: 2, availableTargetUnits: 4_000 } }, 'source', 'target', { pollIntervalMs: 1, timeoutMs: 100 } ) assert.equal(attempts, 3) assert.equal(capacity.availableTargetUnits, 4_000) await assert.rejects( waitForEvacuationCapacity(async () => { throw new Error('/v1/admin/evacuation-capacity failed: forbidden') }, 'source', 'target'), /forbidden/ ) }) test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => { for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value) for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) { assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/) } assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined) assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined) assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1') }) test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => { const harness = directorHarness() harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3' await deployDirector({}, 'candidate-new', harness.operations) for (const revision of ['relay-00002-new', 'relay-00003-new']) { assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3') } }) test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => { const harness = directorHarness() await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' }, 'candidate-new', harness.operations), /exact disabled regional-rehome generation/) assert.equal(harness.state.activeRevision, 'relay-00001-old') assert.equal(harness.state.nextRevision, 2) await deployDirector({}, 'candidate-new', harness.operations) assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0') }) test('sets a reviewed cohort only behind repeated disabled-control verification', async () => { const harness = directorHarness() let verified = 0 const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' } await deployDirector(config, 'candidate-new', { ...harness.operations, assertRegionalRehomeDisabled: async () => { verified++ } }) assert.ok(verified >= 2) assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1') }) test('director deploy refuses a traffic move between reserve-support revisions while a cell is reserve', async () => { const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' const oldServing = directorHarness({ reserveCells: { [candidateOrigin]: ['production-gce-c3'] } }) await assert.rejects( deployDirector({}, 'candidate-new', oldServing.operations), /differ in reserve-mode support while cells are reserve \(production-gce-c3\)/ ) assert.equal(oldServing.state.activeRevision, 'relay-00001-old') const rollbackToOld = directorHarness({ reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'] } }) await assert.rejects(deployDirector({}, 'candidate-new', rollbackToOld.operations), /flip them to db first/) assert.equal(rollbackToOld.state.activeRevision, 'relay-00001-old') const allDb = directorHarness({ reserveCells: { [candidateOrigin]: [] } }) await deployDirector({}, 'candidate-new', allDb.operations) assert.equal(allDb.state.activeRevision, 'relay-00003-new') }) test('two revisions that both read the census may move traffic with reserve cells', () => { assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(['c3'], ['c3'])) assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(undefined, undefined)) assert.throws(() => assertReserveCompatibleTrafficMove(undefined, ['c3']), /flip them to db first/) }) test('reads a census-less director as unsupported and fails closed on any other refusal', async () => { const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' } try { assert.equal( await readReserveCells(config, 'https://old.example.test', ['c1'], async () => new Response('404 Not Found', { status: 404 }) ), undefined ) const modes = { c1: 'db', c2: 'reserve' } assert.deepEqual( await readReserveCells(config, 'https://new.example.test', ['c1', 'c2'], async (url, init) => { assert.equal(url, 'https://new.example.test/v1/admin/cell-admit-mode') const { cellId } = JSON.parse(init.body) return new Response(JSON.stringify({ v: 1, cellId, admitMode: modes[cellId], updatedAt: 1 })) }), ['c2'] ) await assert.rejects( readReserveCells(config, 'https://new.example.test', ['c1'], async () => new Response(JSON.stringify({ error: 'invalid_token' }), { status: 401 }) ), /cell admit mode read failed for c1: invalid_token/ ) } finally { if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment } }) test('a same-image deploy reads no admit modes, and keeps the serving reserve placement', async () => { const servingImageDigest = `sha256:${'f'.repeat(64)}` const harness = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' } }) let reads = 0 harness.operations.readReserveCells = async () => { reads += 1 return [] } await deployDirector({ image: `relay@${servingImageDigest}` }, 'candidate-new', harness.operations) assert.equal(reads, 0) assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'on') }) test('turning reserve placement off is refused while any cell is reserve, even on the same image', async () => { const servingImageDigest = `sha256:${'f'.repeat(64)}` const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' const reserved = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] }, placements: { 'https://relay-hash-uc.a.run.app': 'on', [candidateOrigin]: 'off' } }) await assert.rejects( deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', reserved.operations), /reserve placement would turn off while cells are reserve \(production-gce-c3\); flip them to db first/ ) assert.equal(reserved.state.activeRevision, 'relay-00001-old') const allDb = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, reserveCells: { 'https://relay-hash-uc.a.run.app': [], [candidateOrigin]: [] } }) await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', allDb.operations) assert.equal(allDb.state.revisions.get(allDb.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off') }) test('--reserve-placement takes only a known mode, and only for a director', () => { const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'a'.repeat(64)}`, '--release-id', '1'] assert.equal(parseArguments([...base, '--role', 'director', '--reserve-placement', 'on'])['reserve-placement'], 'on') assert.throws(() => parseArguments([...base, '--role', 'director', '--reserve-placement', 'yes']), /must be preserve, off, dry-run or on/) assert.throws( () => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--reserve-placement', 'on']), /require --role director/ ) assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'preserve' })[DIRECTOR_RESERVE_PLACEMENT_ENV], undefined) assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'dry-run' })[DIRECTOR_RESERVE_PLACEMENT_ENV], 'dry-run') }) test('the guard reads placement as built, so a director that never placed does not block turning it off', async () => { const servingImageDigest = `sha256:${'f'.repeat(64)}` const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' // Configured on, but the serving director reports off: it was never feeding the dead-man. const harness = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] }, placements: { 'https://relay-hash-uc.a.run.app': 'off', [candidateOrigin]: 'off' } }) await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations) assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off') }) test('a shadow-cell change on the same image is checked, and a shrink past a reserve cell is refused', () => { assert.throws( () => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', ['production-gce-c3']), /would drop reserve cells \(production-gce-c3\); flip them to db first/ ) assert.throws(() => assertShadowCellsKeepReserveCells('all', 'production-gce-c1', ['production-gce-c3']), /production-gce-c3/) assert.throws(() => assertShadowCellsKeepReserveCells('all', undefined, ['production-gce-c3']), /production-gce-c3/) assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'all', ['production-gce-c3'])) assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', [])) // A reserve cell the serving list never polled is not being dropped by this deploy. assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'production-gce-c1', ['production-gce-c3'])) }) test('reads a director placement from runtime-status, and reports an image without the field', async () => { const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' } try { const answer = (body) => async () => new Response(JSON.stringify({ v: 1, role: 'director', ...body })) assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'on' })), 'on') assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({})), 'unreported') await assert.rejects(readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'maybe' })), /reports reserve placement maybe/) await assert.rejects( readDirectorPlacement(config, 'https://d.example.test', async () => new Response('{}', { status: 401 })), /runtime status failed/ ) } finally { if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment } }) // Why: staging drifted to a director that trusts gha-deploy while every staging workflow // authenticates as gha-relay; the deploy, not Terraform, owns the live value. test('a director deploy sets the trusted deploy identity only when asked', () => { const config = { project: 'onorca-cloud-staging' } const relay = 'orca-cloud-staging-gha-relay@onorca-cloud-staging.iam.gserviceaccount.com' assert.equal( directorDeploymentEnvironment({ ...config, 'deploy-service-account': relay }) .ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT, relay ) assert.equal( 'ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT' in directorDeploymentEnvironment(config), false ) assert.throws( () => directorDeploymentEnvironment({ ...config, 'deploy-service-account': 'foreign@other-project.iam.gserviceaccount.com' }), /selected project/ ) }) // Staging's director trusted gha-deploy while the workflow authenticated as gha-relay: a bare 401. test('the guard names the deploy identity a drifted director trusts instead of reading into a 401', async () => { const servingImageDigest = `sha256:${'f'.repeat(64)}` const harness = directorHarness({ servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-deploy@p.iam.gserviceaccount.com' }, callerEmail: 'gha-relay@p.iam.gserviceaccount.com' }) await assert.rejects( deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations), /serving director trusts gha-deploy@p\.iam\.gserviceaccount\.com for deploy admin reads, but this deploy authenticates as gha-relay@p\.iam\.gserviceaccount\.com/ ) const trusted = directorHarness({ servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-relay@p.iam.gserviceaccount.com' }, callerEmail: 'gha-relay@p.iam.gserviceaccount.com' }) await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', trusted.operations) }) test('a director deploy takes --admin-audience alone for the reserve guard', () => { const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'e'.repeat(64)}`, '--role', 'director', '--release-id', 'x'] const audience = ['--admin-audience', 'https://relay.example.test'] assert.equal(parseArguments([...base, ...audience])['admin-audience'], 'https://relay.example.test') assert.throws( () => parseArguments([...base, '--expected-rehome-generation', '1', '--rehome-control-origin', 'https://relay.example.test']), /durable rehome verification arguments must be configured together/ ) assert.throws( () => parseArguments([...base, ...audience, '--expected-rehome-generation', '1']), /durable rehome verification arguments must be configured together/ ) }) test('--shadow-seat-feed-cells sets the director setting, and preserve keeps the serving one', async () => { assert.equal( directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'c1, c2' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'c1,c2' ) assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'all' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'all') assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'preserve' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], undefined) assert.throws(() => directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'C1;x' }), /must be preserve, all, or a comma list/) const servingImageDigest = `sha256:${'f'.repeat(64)}` const harness = directorHarness({ servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' } }) await deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'preserve' }, 'candidate-new', harness.operations) assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'c1,c3') const shrink = directorHarness({ servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' }, reserveCells: { 'https://relay-hash-uc.a.run.app': ['c3'], 'https://candidate-new---relay-hash-uc.a.run.app': ['c3'] } }) await assert.rejects( deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'c1' }, 'candidate-new', shrink.operations), /would drop reserve cells \(c3\)/ ) })