mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
* fix(relay): director reaches Cloud SQL through a proxy sidecar (staging), gated event-loop stall alert On Cloud Run's built-in /cloudsql socket a connect() blocks the director's whole event loop until the connector's upstream dial ends, 10 s per attempt while the instance starts. Staging froze both directors for 134 s on 2026-10-11 and production four directors for 40-50 s on 2026-09-28. relay_director_cloud_sql_proxy_sidecar runs the Cloud SQL Auth Proxy beside the director on 127.0.0.1:5432, as the GCE cells already do, with its own TCP database URL secret. relay_event_loop_stall_alert adds the event_loop_ms_max metric and a 5 s stall alert. Both default off, so production plans unchanged; staging turns both on. * fix(relay): adopt a reviewed Terraform director shape on deploy; counter-based stall alert; cheaper probed sidecar - deploy-relay-blue-green: --shape-baseline-revision names the 0% revision a Terraform apply created; the deploy preserves that shape instead of the serving one. It must be the latest created, unserved revision on the serving image. Wired into the staging and production director workflows behind ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE. - Event-loop stall alert is now a gated counter metric on eventLoopDelayMsMax>=5000. - Sidecar gets a /liveness probe and 0.25 vCPU (Cloud Run rejects only a total under 1 vCPU with concurrency above 1; checked with validateOnly against the API). - staging.tfvars pins the serving director's concurrency 1000 and timeout 3600 s, so the adopted apply shape does not fall to the 80 / 30 s defaults. * fix(relay): shape adoption refuses identity, topology, pruning and image changes; GOMAXPROCS=1 on the sidecar parseArguments now refuses --shape-baseline-revision with a runtime identity bootstrap, a director topology change or revision pruning, so every caller is covered, and the deploy refuses a shape baseline whose image differs from the serving one. Both director workflows also refuse the combinations up front. The proxy sidecar runs with GOMAXPROCS=1 to match its 0.25 vCPU quota. The long-hold alert comment is back above its resource.
1212 lines
49 KiB
JavaScript
1212 lines
49 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { readFileSync } from 'node:fs'
|
|
import { test } from 'node:test'
|
|
import { fileURLToPath } from 'node:url'
|
|
import {
|
|
activeRevision,
|
|
correctionCohortPercent,
|
|
DIRECTOR_CORRECTION_COHORT_ENV,
|
|
cloudRunTrafficTag,
|
|
DIRECTOR_ADMISSION_ENVIRONMENT,
|
|
DIRECTOR_REGIONAL_PLACEMENT_ENV,
|
|
DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
|
DIRECTOR_REHOME_AUDIENCE_ENV,
|
|
DIRECTOR_REHOME_IDENTITY_ENV,
|
|
assertRegionalRehomeDisabled,
|
|
assertReserveCompatibleTrafficMove,
|
|
assertShadowCellsKeepReserveCells,
|
|
deployDirector,
|
|
DIRECTOR_RESERVE_PLACEMENT_ENV,
|
|
directorDeploymentEnvironment,
|
|
DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV,
|
|
directorCellSetAddition,
|
|
directorStartupProbeArguments,
|
|
directorTopologyChange,
|
|
environmentUpdateValue,
|
|
parseArguments,
|
|
readDirectorPlacement,
|
|
readReserveCells,
|
|
revisionEnvironment,
|
|
revisionSecretEnvironment,
|
|
suppliedAdminIdentityToken,
|
|
taggedRevisionOrigin,
|
|
taggedTraffic,
|
|
trafficTags,
|
|
waitForEvacuationCapacity
|
|
} from './deploy-relay-blue-green.mjs'
|
|
|
|
// Terraform declares these values; audited blue/green stamps the same values without targeting
|
|
// the drifted service, so this contract must fail before either side can silently diverge.
|
|
function terraformDirectorEnvironment(names) {
|
|
const read = (name) =>
|
|
readFileSync(fileURLToPath(new URL(`../../infra/terraform/${name}`, import.meta.url)), 'utf8')
|
|
const relay = read('relay.tf')
|
|
const variables = read('variables.tf')
|
|
const production = read('environments/production.tfvars')
|
|
return Object.fromEntries(
|
|
names.map((name) => {
|
|
const block = new RegExp(`name\\s*=\\s*"${name}"\\s*\\n\\s*value\\s*=\\s*([^\\n]+)`).exec(relay)
|
|
assert.ok(block, `${name} is not set by relay.tf`)
|
|
const variable = /var\.([a-z_]+)/.exec(block[1])
|
|
assert.ok(variable, `${name} is not sourced from a Terraform variable`)
|
|
// An environment override wins over the variable default, as Terraform resolves it.
|
|
const override = new RegExp(`^${variable[1]}\\s*=\\s*(\\S+)`, 'm').exec(production)
|
|
const fallback = new RegExp(
|
|
`variable\\s+"${variable[1]}"[\\s\\S]*?default\\s*=\\s*(\\S+)`
|
|
).exec(variables)
|
|
assert.ok(override || fallback, `${variable[1]} has neither an override nor a default`)
|
|
return [name, String((override ?? fallback)[1]).replace(/"/g, '')]
|
|
})
|
|
)
|
|
}
|
|
|
|
test('director admission environment matches what Terraform deploys', () => {
|
|
const names = Object.keys(DIRECTOR_ADMISSION_ENVIRONMENT)
|
|
assert.deepEqual(terraformDirectorEnvironment(names), { ...DIRECTOR_ADMISSION_ENVIRONMENT })
|
|
|
|
const relay = readFileSync(
|
|
fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)),
|
|
'utf8'
|
|
)
|
|
assert.match(
|
|
relay,
|
|
/name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED"[\s\S]*?secret\s+= google_secret_manager_secret\.relay_regional_placement_enabled\.secret_id[\s\S]*?version = data\.external\.relay_serving_regional_placement_version\.result\.version/
|
|
)
|
|
assert.match(relay, /data "external" "relay_serving_regional_placement_version"/)
|
|
assert.match(relay, /read-relay-serving-regional-placement-version\.mjs/)
|
|
assert.doesNotMatch(relay, /template\[0\]\.containers\[0\]\.env/)
|
|
})
|
|
|
|
test('validates the final stamped-cell admission state', () => {
|
|
const required = [
|
|
'--project',
|
|
'project',
|
|
'--region',
|
|
'region',
|
|
'--service',
|
|
'service',
|
|
'--image',
|
|
'image',
|
|
'--role',
|
|
'cell',
|
|
'--release-id',
|
|
'release',
|
|
'--director-origin',
|
|
'https://relay.example.com',
|
|
'--admin-audience',
|
|
'https://relay.example.com/v1/admin/drain'
|
|
]
|
|
|
|
assert.equal(parseArguments(required)['final-admission'], undefined)
|
|
assert.equal(
|
|
parseArguments([...required, '--final-admission', 'disabled'])['final-admission'],
|
|
'disabled'
|
|
)
|
|
assert.throws(() => parseArguments([...required, '--final-admission', 'sometimes']))
|
|
assert.equal(parseArguments([...required, '--min-instances', '0'])['min-instances'], '0')
|
|
assert.throws(() => parseArguments([...required, '--min-instances', '-1']))
|
|
assert.throws(() =>
|
|
parseArguments([...required, '--capacity-service-account', 'relay@example.com'])
|
|
)
|
|
})
|
|
|
|
test('validates optional director capacity configuration', () => {
|
|
const cells = [
|
|
{
|
|
id: 'staging-gce-c3',
|
|
url: 'https://c3.relay-staging.onorca.dev',
|
|
capacityRequests: 4_000,
|
|
initiallyEnabled: false,
|
|
region: 'us-central1',
|
|
connectionHardCap: 600,
|
|
connectionUnobservedBound: 60
|
|
}
|
|
]
|
|
const config = {
|
|
project: 'onorca-cloud-staging',
|
|
'capacity-service-account':
|
|
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com',
|
|
'director-cells-json': JSON.stringify(cells)
|
|
}
|
|
assert.deepEqual(directorDeploymentEnvironment(config), {
|
|
...DIRECTOR_ADMISSION_ENVIRONMENT,
|
|
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3',
|
|
ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT:
|
|
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com',
|
|
ORCA_RELAY_CELLS_JSON: JSON.stringify([
|
|
{
|
|
id: cells[0].id,
|
|
url: cells[0].url,
|
|
capacityRequests: cells[0].capacityRequests,
|
|
region: cells[0].region,
|
|
initiallyEnabled: cells[0].initiallyEnabled,
|
|
connectionHardCap: cells[0].connectionHardCap,
|
|
connectionUnobservedBound: cells[0].connectionUnobservedBound
|
|
}
|
|
])
|
|
})
|
|
assert.throws(
|
|
() =>
|
|
directorDeploymentEnvironment({
|
|
...config,
|
|
'capacity-service-account': 'foreign@other-project.iam.gserviceaccount.com'
|
|
}),
|
|
/selected project/
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
directorDeploymentEnvironment({
|
|
...config,
|
|
'director-cells-json': JSON.stringify([{ ...cells[0], unexpected: true }])
|
|
}),
|
|
/invalid cell/
|
|
)
|
|
assert.match(
|
|
environmentUpdateValue(directorDeploymentEnvironment(config)),
|
|
/^\^~\^ORCA_RELAY_DATABASE_POOL_MAX=/
|
|
)
|
|
assert.equal(environmentUpdateValue({ FIRST: 'one', SECOND: 'two' }), 'FIRST=one,SECOND=two')
|
|
assert.deepEqual(
|
|
directorTopologyChange(
|
|
JSON.stringify([
|
|
{
|
|
capacityRequests: 4_000,
|
|
connectionHardCap: 600,
|
|
connectionUnobservedBound: 60,
|
|
id: 'staging-gce-c3',
|
|
initiallyEnabled: false,
|
|
region: 'us-central1',
|
|
url: 'https://c3.relay-staging.onorca.dev'
|
|
}
|
|
]),
|
|
JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]),
|
|
'staging-gce-c3'
|
|
),
|
|
{
|
|
changed: true,
|
|
value: directorDeploymentEnvironment({
|
|
'director-cells-json': JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }])
|
|
}).ORCA_RELAY_CELLS_JSON
|
|
}
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
directorTopologyChange(
|
|
JSON.stringify(cells),
|
|
JSON.stringify([{ ...cells[0], url: 'https://wrong.relay-staging.onorca.dev' }]),
|
|
'staging-gce-c3'
|
|
),
|
|
/outside the reviewed capacity pair/
|
|
)
|
|
})
|
|
|
|
test('validates exact director runtime and regional rehome identities', () => {
|
|
const base = [
|
|
'--project',
|
|
'onorca-cloud',
|
|
'--region',
|
|
'us-central1',
|
|
'--service',
|
|
'orca-cloud-relay',
|
|
'--image',
|
|
`relay@sha256:${'a'.repeat(64)}`,
|
|
'--role',
|
|
'director',
|
|
'--release-id',
|
|
'rehome',
|
|
'--runtime-service-account',
|
|
'relay-director@onorca-cloud.iam.gserviceaccount.com',
|
|
'--rehome-director-service-account',
|
|
'relay-director@onorca-cloud.iam.gserviceaccount.com',
|
|
'--rehome-audience',
|
|
'https://relay.onorca.dev/v1/admin/host-drain',
|
|
'--expected-rehome-generation',
|
|
'7',
|
|
'--rehome-control-origin',
|
|
'https://relay.onorca.dev',
|
|
'--admin-audience',
|
|
'https://relay.onorca.dev/v1/admin/drain'
|
|
]
|
|
const config = parseArguments(base)
|
|
assert.deepEqual(directorDeploymentEnvironment(config), {
|
|
...DIRECTOR_ADMISSION_ENVIRONMENT,
|
|
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3',
|
|
ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`,
|
|
[DIRECTOR_REHOME_IDENTITY_ENV]:
|
|
'relay-director@onorca-cloud.iam.gserviceaccount.com',
|
|
[DIRECTOR_REHOME_AUDIENCE_ENV]:
|
|
'https://relay.onorca.dev/v1/admin/host-drain'
|
|
})
|
|
const missingAudience = [...base]
|
|
missingAudience.splice(missingAudience.indexOf('--rehome-audience'), 2)
|
|
assert.throws(() => parseArguments(missingAudience), /configured together/)
|
|
const invalidOrigin = [...base]
|
|
invalidOrigin[invalidOrigin.indexOf('--rehome-control-origin') + 1] =
|
|
'http://relay.onorca.dev'
|
|
assert.throws(
|
|
() => parseArguments(invalidOrigin),
|
|
/HTTPS origin/
|
|
)
|
|
const mutableImage = [...base]
|
|
mutableImage[mutableImage.indexOf('--image') + 1] = 'relay:latest'
|
|
assert.throws(() => parseArguments(mutableImage), /immutable digest/)
|
|
})
|
|
|
|
test('requires durable regional rehome control to be disabled at the exact generation', async () => {
|
|
const config = {
|
|
'admin-audience': 'https://relay.onorca.dev/v1/admin/drain',
|
|
'expected-rehome-generation': '7'
|
|
}
|
|
const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
|
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc'
|
|
try {
|
|
const control = await assertRegionalRehomeDisabled(
|
|
config,
|
|
'https://candidate.example.test',
|
|
async (url, init) => {
|
|
assert.equal(url, 'https://candidate.example.test/v1/admin/regional-rehome-control')
|
|
assert.equal(init.headers.authorization, 'Bearer aaa.bbb.ccc')
|
|
return new Response(JSON.stringify({
|
|
v: 1,
|
|
control: { generation: 7, enabled: false }
|
|
}))
|
|
}
|
|
)
|
|
assert.equal(control.generation, 7)
|
|
await assert.rejects(
|
|
assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () =>
|
|
new Response(JSON.stringify({
|
|
v: 1,
|
|
control: { generation: 8, enabled: false }
|
|
}))
|
|
),
|
|
/expected generation/
|
|
)
|
|
await assert.rejects(
|
|
assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () =>
|
|
new Response(JSON.stringify({
|
|
v: 1,
|
|
control: { generation: 7, enabled: true }
|
|
}))
|
|
),
|
|
/durably disabled/
|
|
)
|
|
} finally {
|
|
if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
|
else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment
|
|
}
|
|
})
|
|
|
|
test('rejects literal regional placement changes outside the runtime-setting step', () => {
|
|
const base = {
|
|
project: 'onorca-cloud',
|
|
region: 'us-central1',
|
|
service: 'orca-cloud-relay',
|
|
image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`,
|
|
role: 'director',
|
|
'release-id': 'regional-kill-switch'
|
|
}
|
|
const args = Object.entries(base).flatMap(([key, value]) => [`--${key}`, value])
|
|
|
|
assert.doesNotThrow(() => parseArguments(args))
|
|
assert.throws(
|
|
() => parseArguments([...args, '--regional-placement-enabled', 'false']),
|
|
/audited runtime-setting step/
|
|
)
|
|
})
|
|
|
|
test('appends disabled Asia cells without changing the existing director topology', () => {
|
|
const current = [
|
|
{
|
|
id: 'production-gce-c26',
|
|
url: 'https://c26.relay.onorca.dev',
|
|
capacityRequests: 4_000,
|
|
initiallyEnabled: true,
|
|
connectionHardCap: 1_000,
|
|
connectionUnobservedBound: 60
|
|
}
|
|
]
|
|
const asia = {
|
|
id: 'production-gce-c27',
|
|
url: 'https://c27.relay.onorca.dev',
|
|
region: 'asia-east2',
|
|
capacityRequests: 6_000,
|
|
initiallyEnabled: false,
|
|
connectionHardCap: 3_000,
|
|
connectionUnobservedBound: 60
|
|
}
|
|
|
|
assert.deepEqual(
|
|
directorCellSetAddition(
|
|
JSON.stringify(current),
|
|
JSON.stringify([asia, { ...current[0], region: 'us-central1' }])
|
|
),
|
|
{
|
|
changed: true,
|
|
value: directorDeploymentEnvironment({
|
|
'director-cells-json': JSON.stringify([asia, { ...current[0], region: 'us-central1' }])
|
|
}).ORCA_RELAY_CELLS_JSON
|
|
}
|
|
)
|
|
const exact = JSON.stringify([{ ...current[0], region: 'us-central1' }, asia])
|
|
assert.deepEqual(directorCellSetAddition(exact, exact), {
|
|
changed: false,
|
|
value: directorDeploymentEnvironment({ 'director-cells-json': exact })
|
|
.ORCA_RELAY_CELLS_JSON
|
|
})
|
|
assert.throws(
|
|
() => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1', capacityRequests: 6_000 }, asia])),
|
|
/changes an existing cell/
|
|
)
|
|
assert.throws(
|
|
() => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1' }, { ...asia, initiallyEnabled: true }])),
|
|
/must start disabled/
|
|
)
|
|
})
|
|
|
|
test('pins a director startup probe above the bounded reconciliation window', () => {
|
|
assert.deepEqual(directorStartupProbeArguments('director'), [
|
|
'--startup-probe',
|
|
'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1'
|
|
])
|
|
assert.deepEqual(directorStartupProbeArguments('cell'), [])
|
|
})
|
|
|
|
test('bounds traffic tags by the Cloud Run service-plus-tag contract', () => {
|
|
const service = 'orca-cloud-relay-staging-c1'
|
|
const candidate = cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a')
|
|
assert.match(candidate, /^candidate-[a-f0-9]{9}$/)
|
|
assert.equal(service.length + candidate.length, 46)
|
|
assert.equal(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a'))
|
|
assert.notEqual(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-2-19cc312a'))
|
|
assert.throws(() => cloudRunTrafficTag(`${service}-too-long`, 'candidate', 'release'))
|
|
})
|
|
|
|
test('derives and validates a Cloud Run tagged revision origin', () => {
|
|
assert.equal(
|
|
taggedRevisionOrigin(
|
|
'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app',
|
|
'candidate-123'
|
|
),
|
|
'https://candidate-123---orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app'
|
|
)
|
|
assert.throws(() => taggedRevisionOrigin('https://relay-staging.onorca.dev', 'candidate-123'))
|
|
assert.throws(() =>
|
|
taggedRevisionOrigin(
|
|
'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app',
|
|
'123-invalid'
|
|
)
|
|
)
|
|
})
|
|
|
|
test('requires exactly one active revision and reads queried tag metadata', () => {
|
|
const service = {
|
|
status: {
|
|
traffic: [
|
|
{ percent: 100, revisionName: 'relay-00001-old' },
|
|
{
|
|
percent: 0,
|
|
revisionName: 'relay-00002-new',
|
|
tag: 'candidate-123',
|
|
url: 'https://candidate-123---relay-hash-uc.a.run.app'
|
|
}
|
|
]
|
|
}
|
|
}
|
|
assert.equal(activeRevision(service), 'relay-00001-old')
|
|
assert.deepEqual(taggedTraffic(service, 'candidate-123'), {
|
|
origin: 'https://candidate-123---relay-hash-uc.a.run.app',
|
|
revision: 'relay-00002-new'
|
|
})
|
|
assert.throws(() =>
|
|
activeRevision({ status: { traffic: [{ percent: 50 }, { percent: 50 }] } })
|
|
)
|
|
assert.deepEqual(trafficTags(service), ['candidate-123'])
|
|
})
|
|
|
|
const adminToken = (email) =>
|
|
`e30.${Buffer.from(JSON.stringify(email ? { email } : {})).toString('base64url')}.sig`
|
|
|
|
function directorHarness({
|
|
deployFailure,
|
|
deleteFailure,
|
|
cleanupReportsFailure = false,
|
|
cleanupFailsBeforeRemoval = false,
|
|
servingMinimum = 1,
|
|
servingMaximum = 5,
|
|
// Reproduces gcloud dropping minScale from a newly created revision.
|
|
dropRequestedMinimum = false,
|
|
servingServiceAccount,
|
|
servingImageDigest = `sha256:${'f'.repeat(64)}`,
|
|
reserveCells = {},
|
|
servingEnvironment = {},
|
|
placements = {},
|
|
callerEmail,
|
|
// The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one.
|
|
templateTransport
|
|
} = {}) {
|
|
const state = {
|
|
activeRevision: 'relay-00001-old',
|
|
tags: new Map([['candidate-old', 'relay-00000-stale']]),
|
|
revisions: new Map([
|
|
['relay-00000-stale', { env: { ORCA_RELAY_ROLE: 'director' }, minimum: 1, maximum: 5 }],
|
|
[
|
|
'relay-00001-old',
|
|
{
|
|
env: {
|
|
ORCA_RELAY_ROLE: 'director',
|
|
...servingEnvironment
|
|
},
|
|
secrets: {
|
|
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
|
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
|
version: '1'
|
|
}
|
|
},
|
|
minimum: servingMinimum,
|
|
maximum: servingMaximum,
|
|
serviceAccount: servingServiceAccount,
|
|
image: `relay@${servingImageDigest}`
|
|
}
|
|
]
|
|
]),
|
|
nextRevision: 2,
|
|
latestCreated: 'relay-00001-old',
|
|
templateTransport
|
|
}
|
|
const removed = []
|
|
const healthProtocols = []
|
|
let pendingCleanupFailure = cleanupFailsBeforeRemoval
|
|
const operations = {
|
|
describeService: () => ({
|
|
status: {
|
|
url: 'https://relay-hash-uc.a.run.app',
|
|
latestCreatedRevisionName: state.latestCreated,
|
|
traffic: [
|
|
{ percent: 100, revisionName: state.activeRevision },
|
|
...[...state.tags].map(([tag, revisionName]) => ({
|
|
tag,
|
|
revisionName,
|
|
url: `https://${tag}---relay-hash-uc.a.run.app`
|
|
}))
|
|
]
|
|
}
|
|
}),
|
|
describeRevision: (_config, revision) => {
|
|
const value = state.revisions.get(revision)
|
|
return {
|
|
metadata: {
|
|
annotations: {
|
|
'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0),
|
|
'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5),
|
|
...value?.transport?.annotations
|
|
}
|
|
},
|
|
spec: {
|
|
serviceAccountName: value?.serviceAccount,
|
|
containers: [
|
|
{
|
|
name: value?.transport?.name,
|
|
image: value?.image,
|
|
env: [
|
|
...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })),
|
|
...Object.entries(value?.secrets ?? {}).map(([name, secretKeyRef]) => ({
|
|
name,
|
|
valueSource: { secretKeyRef }
|
|
}))
|
|
]
|
|
},
|
|
...value?.transport?.sidecars ?? []
|
|
]
|
|
}
|
|
}
|
|
},
|
|
deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => {
|
|
const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new`
|
|
state.revisions.set(revision, {
|
|
// gcloud --update-env-vars keeps every variable the deploy does not name.
|
|
env: { ORCA_RELAY_ROLE: 'director', ...servingEnvironment, ...env },
|
|
secrets: {
|
|
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
|
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
|
version: regionalVersion
|
|
}
|
|
},
|
|
minimum: dropRequestedMinimum ? 0 : Number(minimum ?? 1),
|
|
maximum,
|
|
serviceAccount:
|
|
config['runtime-service-account'] ??
|
|
state.revisions.get(state.activeRevision)?.serviceAccount,
|
|
image: image ?? state.revisions.get(state.activeRevision)?.image,
|
|
transport: state.templateTransport
|
|
})
|
|
state.latestCreated = revision
|
|
state.tags.set(tag, revision)
|
|
if (deployFailure) throw deployFailure
|
|
},
|
|
listRevisions: () =>
|
|
[...state.revisions].map(([name]) => ({ metadata: { name } })),
|
|
deleteRevision: (_config, revision) => {
|
|
if (deleteFailure) throw deleteFailure
|
|
state.revisions.delete(revision)
|
|
},
|
|
updateTraffic: (_config, args) => {
|
|
const remove = args.find((argument) => argument.startsWith('--remove-tags='))
|
|
if (remove) {
|
|
const tags = remove.slice('--remove-tags='.length).split(',')
|
|
removed.push(tags)
|
|
if (pendingCleanupFailure && tags.includes('candidate-new')) {
|
|
pendingCleanupFailure = false
|
|
throw new Error('failed to remove promoted candidate tag')
|
|
}
|
|
for (const tag of tags) state.tags.delete(tag)
|
|
if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision')
|
|
return
|
|
}
|
|
const promote = args.find((argument) => argument.startsWith('--to-tags='))
|
|
assert.ok(promote)
|
|
const tag = promote.slice('--to-tags='.length).split('=')[0]
|
|
state.activeRevision = state.tags.get(tag)
|
|
},
|
|
waitForHealth: async (_origin, connectionCapacityProtocol) => {
|
|
healthProtocols.push(connectionCapacityProtocol)
|
|
},
|
|
readReserveCells: async (_config, origin) => reserveCells[origin],
|
|
readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported',
|
|
adminIdentityToken: () => adminToken(callerEmail)
|
|
}
|
|
return { state, removed, healthProtocols, operations }
|
|
}
|
|
|
|
test('director deploy removes stale and promoted Cloud Run tags', async () => {
|
|
const harness = directorHarness()
|
|
const config = {
|
|
project: 'onorca-cloud-staging',
|
|
'capacity-service-account':
|
|
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
|
|
}
|
|
await deployDirector(config, 'candidate-new', harness.operations)
|
|
assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']])
|
|
assert.equal(harness.state.activeRevision, 'relay-00003-new')
|
|
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
|
assert.deepEqual([...harness.state.revisions.keys()], [
|
|
'relay-00000-stale',
|
|
'relay-00001-old',
|
|
'relay-00002-new',
|
|
'relay-00003-new'
|
|
])
|
|
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
|
|
assert.deepEqual(
|
|
Object.fromEntries(
|
|
Object.entries(harness.state.revisions.get(revision).env).filter(([key]) =>
|
|
key in DIRECTOR_ADMISSION_ENVIRONMENT
|
|
)
|
|
),
|
|
DIRECTOR_ADMISSION_ENVIRONMENT
|
|
)
|
|
assert.equal(
|
|
harness.state.revisions.get(revision).env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT,
|
|
config['capacity-service-account']
|
|
)
|
|
}
|
|
assert.deepEqual(harness.healthProtocols, [undefined, undefined])
|
|
})
|
|
|
|
test('director deploy stamps the durable regional placement secret reference', async () => {
|
|
const harness = directorHarness()
|
|
await deployDirector({}, 'candidate-new', harness.operations)
|
|
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
|
|
assert.deepEqual(
|
|
harness.state.revisions.get(revision).secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV],
|
|
{ secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: '1' }
|
|
)
|
|
}
|
|
})
|
|
|
|
test('bootstraps both rollback and candidate onto the distinct director identity', async () => {
|
|
const predecessor = 'relay-runtime@onorca-cloud.iam.gserviceaccount.com'
|
|
const director = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
|
|
const harness = directorHarness({ servingServiceAccount: predecessor })
|
|
const inspected = []
|
|
harness.operations.assertRegionalRehomeDisabled = async (_config, origin) => {
|
|
inspected.push(origin)
|
|
}
|
|
await deployDirector({
|
|
project: 'onorca-cloud',
|
|
'runtime-service-account': director,
|
|
'predecessor-runtime-service-account': predecessor,
|
|
'predecessor-image-digest': `sha256:${'f'.repeat(64)}`,
|
|
'bootstrap-runtime-identity': 'true',
|
|
'expected-rehome-generation': '0',
|
|
'rehome-control-origin': 'https://relay.onorca.dev'
|
|
}, 'candidate-new', harness.operations)
|
|
assert.equal(
|
|
harness.state.revisions.get(harness.state.activeRevision).serviceAccount,
|
|
director
|
|
)
|
|
assert.equal(
|
|
harness.state.revisions.get(harness.state.tags.get('selector-rollback')).serviceAccount,
|
|
director
|
|
)
|
|
assert.deepEqual(inspected, [
|
|
'https://selector-rollback---relay-hash-uc.a.run.app',
|
|
'https://candidate-new---relay-hash-uc.a.run.app'
|
|
])
|
|
})
|
|
|
|
test('steady-state director deploy rejects the predecessor identity', async () => {
|
|
const harness = directorHarness({
|
|
servingServiceAccount: 'relay-runtime@onorca-cloud.iam.gserviceaccount.com'
|
|
})
|
|
await assert.rejects(deployDirector({
|
|
'runtime-service-account': 'relay-director@onorca-cloud.iam.gserviceaccount.com'
|
|
}, 'candidate-new', harness.operations), /unexpected runtime service account/)
|
|
})
|
|
|
|
test('director deploy prunes old revisions when requested', async () => {
|
|
const harness = directorHarness()
|
|
await deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations)
|
|
assert.deepEqual([...harness.state.revisions.keys()], [
|
|
'relay-00002-new',
|
|
'relay-00003-new'
|
|
])
|
|
assert.deepEqual(harness.healthProtocols, [2, 2])
|
|
})
|
|
|
|
test('a prune failure preserves the active and rollback traffic pair', async () => {
|
|
const harness = directorHarness({ deleteFailure: new Error('injected delete failure') })
|
|
await assert.rejects(
|
|
deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations),
|
|
/injected delete failure/
|
|
)
|
|
assert.equal(harness.state.activeRevision, 'relay-00003-new')
|
|
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
|
})
|
|
|
|
test('director deploy removes a candidate tag left by a failed update', async () => {
|
|
const failure = new Error('candidate failed to become ready')
|
|
const harness = directorHarness({ deployFailure: failure })
|
|
await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure)
|
|
assert.deepEqual(harness.removed, [['candidate-old'], ['selector-rollback']])
|
|
assert.equal(harness.state.tags.size, 0)
|
|
})
|
|
|
|
test('director candidate inherits the serving warm-instance floor', async () => {
|
|
const harness = directorHarness({ servingMinimum: 5 })
|
|
await deployDirector({}, 'candidate-new', harness.operations)
|
|
const serving = harness.state.revisions.get(harness.state.activeRevision)
|
|
assert.equal(serving.minimum, 5)
|
|
// The standby rollback revision must stay cold.
|
|
const rollback = harness.state.revisions.get(harness.state.tags.get('selector-rollback'))
|
|
assert.equal(rollback.minimum, 0)
|
|
})
|
|
|
|
test('director deploy rejects a serving maximum above the checked budget', async () => {
|
|
const harness = directorHarness({ servingMaximum: 6 })
|
|
await assert.rejects(
|
|
deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations),
|
|
/holds 6 maximum instances, expected 5/
|
|
)
|
|
})
|
|
|
|
test('an explicit --min-instances still overrides the serving floor', async () => {
|
|
const harness = directorHarness({ servingMinimum: 5 })
|
|
await deployDirector({ 'min-instances': '0' }, 'candidate-new', harness.operations)
|
|
assert.equal(harness.state.revisions.get(harness.state.activeRevision).minimum, 0)
|
|
})
|
|
|
|
test('director deploy refuses to move traffic onto a candidate that lost the floor', async () => {
|
|
const harness = directorHarness({ servingMinimum: 5, dropRequestedMinimum: true })
|
|
await assert.rejects(
|
|
deployDirector({}, 'candidate-new', harness.operations),
|
|
/candidate holds 0 minimum instances, expected 5/
|
|
)
|
|
// Traffic never moved, so the original revision still serves.
|
|
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
|
})
|
|
|
|
const SERVING_IMAGE = `relay@sha256:${'f'.repeat(64)}`
|
|
|
|
// What a Terraform apply of relay_director_cloud_sql_proxy_sidecar puts in the template.
|
|
const SIDECAR_TRANSPORT = {
|
|
name: 'relay',
|
|
annotations: { 'run.googleapis.com/container-dependencies': '{"relay":["cloud-sql-proxy"]}' },
|
|
sidecars: [{ name: 'cloud-sql-proxy', image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:1' }]
|
|
}
|
|
|
|
// The apply leaves its revision at 0%, as the director service declares no traffic block.
|
|
function applySidecarTemplate(harness) {
|
|
const serving = harness.state.revisions.get(harness.state.activeRevision)
|
|
harness.state.templateTransport = SIDECAR_TRANSPORT
|
|
harness.state.revisions.set('relay-00009-terraform', { ...serving, transport: SIDECAR_TRANSPORT })
|
|
harness.state.latestCreated = 'relay-00009-terraform'
|
|
}
|
|
|
|
test('a deploy after a sidecar apply refuses the new shape unless the apply revision is named', async () => {
|
|
const harness = directorHarness()
|
|
applySidecarTemplate(harness)
|
|
await assert.rejects(
|
|
deployDirector({}, 'candidate-new', harness.operations),
|
|
/unrelated revision shape/
|
|
)
|
|
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
|
})
|
|
|
|
test('a deploy adopts the shape of the named Terraform apply revision', async () => {
|
|
const harness = directorHarness()
|
|
applySidecarTemplate(harness)
|
|
await deployDirector(
|
|
{ image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' },
|
|
'candidate-new',
|
|
harness.operations
|
|
)
|
|
const active = harness.state.revisions.get(harness.state.activeRevision)
|
|
assert.equal(active.transport, SIDECAR_TRANSPORT)
|
|
assert.equal(
|
|
harness.state.revisions.get(harness.state.tags.get('selector-rollback')).transport,
|
|
SIDECAR_TRANSPORT
|
|
)
|
|
})
|
|
|
|
test('the named shape baseline must be the unserved latest revision on the serving image', async () => {
|
|
const stale = directorHarness()
|
|
applySidecarTemplate(stale)
|
|
await assert.rejects(
|
|
deployDirector({ 'shape-baseline-revision': 'relay-00000-stale' }, 'candidate-new', stale.operations),
|
|
/not the latest created revision/
|
|
)
|
|
const serving = directorHarness()
|
|
await assert.rejects(
|
|
deployDirector({ 'shape-baseline-revision': 'relay-00001-old' }, 'candidate-new', serving.operations),
|
|
/already serving/
|
|
)
|
|
const otherImage = directorHarness()
|
|
applySidecarTemplate(otherImage)
|
|
otherImage.state.revisions.get('relay-00009-terraform').image = `relay@sha256:${'e'.repeat(64)}`
|
|
await assert.rejects(
|
|
deployDirector({ image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, 'candidate-new', otherImage.operations),
|
|
/does not run the serving image/
|
|
)
|
|
const newImage = directorHarness()
|
|
applySidecarTemplate(newImage)
|
|
await assert.rejects(
|
|
deployDirector(
|
|
{ image: `relay@sha256:${'e'.repeat(64)}`, 'shape-baseline-revision': 'relay-00009-terraform' },
|
|
'candidate-new',
|
|
newImage.operations
|
|
),
|
|
/must keep the serving image/
|
|
)
|
|
for (const harness of [stale, serving, otherImage, newImage]) {
|
|
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
|
}
|
|
})
|
|
|
|
test('--shape-baseline-revision takes only a revision name, and only for a director', () => {
|
|
const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', 'i@sha256:' + 'a'.repeat(64), '--release-id', 'x']
|
|
assert.throws(
|
|
() => parseArguments([...base, '--role', 'director', '--shape-baseline-revision', 'Bad_Name']),
|
|
/must be a Cloud Run revision name/
|
|
)
|
|
assert.throws(
|
|
() => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--shape-baseline-revision', 'relay-1']),
|
|
/--shape-baseline-revision requires --role director/
|
|
)
|
|
const director = [...base, '--role', 'director', '--shape-baseline-revision', 'relay-1']
|
|
assert.equal(parseArguments(director)['shape-baseline-revision'], 'relay-1')
|
|
for (const [extra, refusal] of [
|
|
[['--bootstrap-runtime-identity', 'true'], /cannot combine with a runtime identity bootstrap/],
|
|
[['--capacity-cell-id', 'c1', '--director-cells-json', '[]'], /cannot combine with a director topology change/],
|
|
[['--director-cells-json', '[]'], /cannot combine with a director topology change/],
|
|
[['--prune-revisions', 'true'], /cannot combine with revision pruning/]
|
|
]) {
|
|
assert.throws(() => parseArguments([...director, ...extra]), refusal)
|
|
}
|
|
})
|
|
|
|
test('director deploy rejects unrelated revision-shape drift', async () => {
|
|
const harness = directorHarness()
|
|
const describeRevision = harness.operations.describeRevision
|
|
harness.operations.describeRevision = (config, revision) => {
|
|
const described = describeRevision(config, revision)
|
|
described.spec.containerConcurrency = revision === 'relay-00001-old' ? 80 : 1_000
|
|
return described
|
|
}
|
|
await assert.rejects(
|
|
deployDirector({}, 'candidate-new', harness.operations),
|
|
/unrelated revision shape/
|
|
)
|
|
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
|
})
|
|
|
|
test('director cleanup verifies success when gcloud reports a stale revision failure', async () => {
|
|
const harness = directorHarness({ cleanupReportsFailure: true })
|
|
await deployDirector({}, 'candidate-new', harness.operations)
|
|
assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']])
|
|
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
|
})
|
|
|
|
test('director deploy restores rollback traffic after promoted-tag cleanup fails', async () => {
|
|
const harness = directorHarness({ cleanupFailsBeforeRemoval: true })
|
|
await assert.rejects(
|
|
deployDirector({}, 'candidate-new', harness.operations),
|
|
/failed to remove promoted candidate tag/
|
|
)
|
|
assert.equal(
|
|
harness.state.activeRevision,
|
|
harness.state.tags.get('selector-rollback')
|
|
)
|
|
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
|
})
|
|
|
|
test('reads only literal revision environment values', () => {
|
|
assert.deepEqual(
|
|
revisionEnvironment({
|
|
spec: {
|
|
containers: [
|
|
{
|
|
env: [
|
|
{ name: 'ORCA_RELAY_CELL_ID', value: 'staging-c1' },
|
|
{ name: 'ORCA_RELAY_CELL_CAPACITY', value: '900' },
|
|
{ name: 'DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database' } } }
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}),
|
|
{ ORCA_RELAY_CELL_ID: 'staging-c1', ORCA_RELAY_CELL_CAPACITY: '900' }
|
|
)
|
|
})
|
|
|
|
test('reads only Secret Manager revision environment references', () => {
|
|
assert.deepEqual(
|
|
revisionSecretEnvironment({
|
|
spec: {
|
|
containers: [{ env: [
|
|
{ name: 'LITERAL', value: 'true' },
|
|
{
|
|
name: DIRECTOR_REGIONAL_PLACEMENT_ENV,
|
|
valueSource: { secretKeyRef: {
|
|
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
|
version: 'latest'
|
|
} }
|
|
},
|
|
{
|
|
name: 'GCP_SECRET_SHAPE',
|
|
valueFrom: { secretKeyRef: { name: 'gcp-secret', key: '2' } }
|
|
}
|
|
] }]
|
|
}
|
|
}),
|
|
{
|
|
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
|
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
|
version: 'latest'
|
|
},
|
|
GCP_SECRET_SHAPE: {
|
|
secret: 'gcp-secret',
|
|
version: '2'
|
|
}
|
|
}
|
|
)
|
|
})
|
|
|
|
test('accepts only a bounded JWT-shaped supplied admin identity token', () => {
|
|
assert.equal(suppliedAdminIdentityToken({}), null)
|
|
assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc')
|
|
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' }))
|
|
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' }))
|
|
assert.throws(() =>
|
|
suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` })
|
|
)
|
|
})
|
|
|
|
test('waits for authenticated target readiness without hiding other capacity errors', async () => {
|
|
let attempts = 0
|
|
const capacity = await waitForEvacuationCapacity(
|
|
async () => {
|
|
attempts += 1
|
|
if (attempts < 3) throw new Error('/v1/admin/evacuation-capacity failed: target_cell_unavailable')
|
|
return { requiredTargetUnits: 2, availableTargetUnits: 4_000 }
|
|
},
|
|
'source',
|
|
'target',
|
|
{ pollIntervalMs: 1, timeoutMs: 100 }
|
|
)
|
|
assert.equal(attempts, 3)
|
|
assert.equal(capacity.availableTargetUnits, 4_000)
|
|
await assert.rejects(
|
|
waitForEvacuationCapacity(async () => {
|
|
throw new Error('/v1/admin/evacuation-capacity failed: forbidden')
|
|
}, 'source', 'target'),
|
|
/forbidden/
|
|
)
|
|
})
|
|
|
|
|
|
test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => {
|
|
for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value)
|
|
for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) {
|
|
assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/)
|
|
}
|
|
assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
|
|
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
|
|
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1')
|
|
})
|
|
|
|
test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => {
|
|
const harness = directorHarness()
|
|
harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3'
|
|
await deployDirector({}, 'candidate-new', harness.operations)
|
|
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
|
|
assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3')
|
|
}
|
|
})
|
|
|
|
test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => {
|
|
const harness = directorHarness()
|
|
await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' },
|
|
'candidate-new', harness.operations), /exact disabled regional-rehome generation/)
|
|
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
|
assert.equal(harness.state.nextRevision, 2)
|
|
await deployDirector({}, 'candidate-new', harness.operations)
|
|
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0')
|
|
})
|
|
|
|
test('sets a reviewed cohort only behind repeated disabled-control verification', async () => {
|
|
const harness = directorHarness()
|
|
let verified = 0
|
|
const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' }
|
|
await deployDirector(config, 'candidate-new', { ...harness.operations,
|
|
assertRegionalRehomeDisabled: async () => { verified++ } })
|
|
assert.ok(verified >= 2)
|
|
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1')
|
|
})
|
|
|
|
test('director deploy refuses a traffic move between reserve-support revisions while a cell is reserve', async () => {
|
|
const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app'
|
|
const oldServing = directorHarness({
|
|
reserveCells: { [candidateOrigin]: ['production-gce-c3'] }
|
|
})
|
|
await assert.rejects(
|
|
deployDirector({}, 'candidate-new', oldServing.operations),
|
|
/differ in reserve-mode support while cells are reserve \(production-gce-c3\)/
|
|
)
|
|
assert.equal(oldServing.state.activeRevision, 'relay-00001-old')
|
|
const rollbackToOld = directorHarness({
|
|
reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'] }
|
|
})
|
|
await assert.rejects(deployDirector({}, 'candidate-new', rollbackToOld.operations), /flip them to db first/)
|
|
assert.equal(rollbackToOld.state.activeRevision, 'relay-00001-old')
|
|
const allDb = directorHarness({ reserveCells: { [candidateOrigin]: [] } })
|
|
await deployDirector({}, 'candidate-new', allDb.operations)
|
|
assert.equal(allDb.state.activeRevision, 'relay-00003-new')
|
|
})
|
|
|
|
test('two revisions that both read the census may move traffic with reserve cells', () => {
|
|
assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(['c3'], ['c3']))
|
|
assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(undefined, undefined))
|
|
assert.throws(() => assertReserveCompatibleTrafficMove(undefined, ['c3']), /flip them to db first/)
|
|
})
|
|
|
|
test('reads a census-less director as unsupported and fails closed on any other refusal', async () => {
|
|
const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
|
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc'
|
|
const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' }
|
|
try {
|
|
assert.equal(
|
|
await readReserveCells(config, 'https://old.example.test', ['c1'], async () =>
|
|
new Response('404 Not Found', { status: 404 })
|
|
),
|
|
undefined
|
|
)
|
|
const modes = { c1: 'db', c2: 'reserve' }
|
|
assert.deepEqual(
|
|
await readReserveCells(config, 'https://new.example.test', ['c1', 'c2'], async (url, init) => {
|
|
assert.equal(url, 'https://new.example.test/v1/admin/cell-admit-mode')
|
|
const { cellId } = JSON.parse(init.body)
|
|
return new Response(JSON.stringify({ v: 1, cellId, admitMode: modes[cellId], updatedAt: 1 }))
|
|
}),
|
|
['c2']
|
|
)
|
|
await assert.rejects(
|
|
readReserveCells(config, 'https://new.example.test', ['c1'], async () =>
|
|
new Response(JSON.stringify({ error: 'invalid_token' }), { status: 401 })
|
|
),
|
|
/cell admit mode read failed for c1: invalid_token/
|
|
)
|
|
} finally {
|
|
if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
|
else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment
|
|
}
|
|
})
|
|
|
|
test('a same-image deploy reads no admit modes, and keeps the serving reserve placement', async () => {
|
|
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
|
const harness = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' } })
|
|
let reads = 0
|
|
harness.operations.readReserveCells = async () => {
|
|
reads += 1
|
|
return []
|
|
}
|
|
await deployDirector({ image: `relay@${servingImageDigest}` }, 'candidate-new', harness.operations)
|
|
assert.equal(reads, 0)
|
|
assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'on')
|
|
})
|
|
|
|
test('turning reserve placement off is refused while any cell is reserve, even on the same image', async () => {
|
|
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
|
const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app'
|
|
const reserved = directorHarness({
|
|
servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' },
|
|
reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] },
|
|
placements: { 'https://relay-hash-uc.a.run.app': 'on', [candidateOrigin]: 'off' }
|
|
})
|
|
await assert.rejects(
|
|
deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', reserved.operations),
|
|
/reserve placement would turn off while cells are reserve \(production-gce-c3\); flip them to db first/
|
|
)
|
|
assert.equal(reserved.state.activeRevision, 'relay-00001-old')
|
|
const allDb = directorHarness({
|
|
servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' },
|
|
reserveCells: { 'https://relay-hash-uc.a.run.app': [], [candidateOrigin]: [] }
|
|
})
|
|
await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', allDb.operations)
|
|
assert.equal(allDb.state.revisions.get(allDb.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off')
|
|
})
|
|
|
|
test('--reserve-placement takes only a known mode, and only for a director', () => {
|
|
const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'a'.repeat(64)}`, '--release-id', '1']
|
|
assert.equal(parseArguments([...base, '--role', 'director', '--reserve-placement', 'on'])['reserve-placement'], 'on')
|
|
assert.throws(() => parseArguments([...base, '--role', 'director', '--reserve-placement', 'yes']), /must be preserve, off, dry-run or on/)
|
|
assert.throws(
|
|
() => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--reserve-placement', 'on']),
|
|
/require --role director/
|
|
)
|
|
assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'preserve' })[DIRECTOR_RESERVE_PLACEMENT_ENV], undefined)
|
|
assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'dry-run' })[DIRECTOR_RESERVE_PLACEMENT_ENV], 'dry-run')
|
|
})
|
|
|
|
test('the guard reads placement as built, so a director that never placed does not block turning it off', async () => {
|
|
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
|
const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app'
|
|
// Configured on, but the serving director reports off: it was never feeding the dead-man.
|
|
const harness = directorHarness({
|
|
servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' },
|
|
reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] },
|
|
placements: { 'https://relay-hash-uc.a.run.app': 'off', [candidateOrigin]: 'off' }
|
|
})
|
|
await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations)
|
|
assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off')
|
|
})
|
|
|
|
test('a shadow-cell change on the same image is checked, and a shrink past a reserve cell is refused', () => {
|
|
assert.throws(
|
|
() => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', ['production-gce-c3']),
|
|
/would drop reserve cells \(production-gce-c3\); flip them to db first/
|
|
)
|
|
assert.throws(() => assertShadowCellsKeepReserveCells('all', 'production-gce-c1', ['production-gce-c3']), /production-gce-c3/)
|
|
assert.throws(() => assertShadowCellsKeepReserveCells('all', undefined, ['production-gce-c3']), /production-gce-c3/)
|
|
assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'all', ['production-gce-c3']))
|
|
assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', []))
|
|
// A reserve cell the serving list never polled is not being dropped by this deploy.
|
|
assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'production-gce-c1', ['production-gce-c3']))
|
|
})
|
|
|
|
test('reads a director placement from runtime-status, and reports an image without the field', async () => {
|
|
const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
|
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc'
|
|
const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' }
|
|
try {
|
|
const answer = (body) => async () => new Response(JSON.stringify({ v: 1, role: 'director', ...body }))
|
|
assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'on' })), 'on')
|
|
assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({})), 'unreported')
|
|
await assert.rejects(readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'maybe' })), /reports reserve placement maybe/)
|
|
await assert.rejects(
|
|
readDirectorPlacement(config, 'https://d.example.test', async () => new Response('{}', { status: 401 })),
|
|
/runtime status failed/
|
|
)
|
|
} finally {
|
|
if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
|
else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment
|
|
}
|
|
})
|
|
|
|
// Why: staging drifted to a director that trusts gha-deploy while every staging workflow
|
|
// authenticates as gha-relay; the deploy, not Terraform, owns the live value.
|
|
test('a director deploy sets the trusted deploy identity only when asked', () => {
|
|
const config = { project: 'onorca-cloud-staging' }
|
|
const relay = 'orca-cloud-staging-gha-relay@onorca-cloud-staging.iam.gserviceaccount.com'
|
|
assert.equal(
|
|
directorDeploymentEnvironment({ ...config, 'deploy-service-account': relay })
|
|
.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT,
|
|
relay
|
|
)
|
|
assert.equal(
|
|
'ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT' in directorDeploymentEnvironment(config),
|
|
false
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
directorDeploymentEnvironment({
|
|
...config,
|
|
'deploy-service-account': 'foreign@other-project.iam.gserviceaccount.com'
|
|
}),
|
|
/selected project/
|
|
)
|
|
})
|
|
|
|
// Staging's director trusted gha-deploy while the workflow authenticated as gha-relay: a bare 401.
|
|
test('the guard names the deploy identity a drifted director trusts instead of reading into a 401', async () => {
|
|
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
|
const harness = directorHarness({
|
|
servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-deploy@p.iam.gserviceaccount.com' },
|
|
callerEmail: 'gha-relay@p.iam.gserviceaccount.com'
|
|
})
|
|
await assert.rejects(
|
|
deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations),
|
|
/serving director trusts gha-deploy@p\.iam\.gserviceaccount\.com for deploy admin reads, but this deploy authenticates as gha-relay@p\.iam\.gserviceaccount\.com/
|
|
)
|
|
const trusted = directorHarness({
|
|
servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-relay@p.iam.gserviceaccount.com' },
|
|
callerEmail: 'gha-relay@p.iam.gserviceaccount.com'
|
|
})
|
|
await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', trusted.operations)
|
|
})
|
|
|
|
test('a director deploy takes --admin-audience alone for the reserve guard', () => {
|
|
const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'e'.repeat(64)}`, '--role', 'director', '--release-id', 'x']
|
|
const audience = ['--admin-audience', 'https://relay.example.test']
|
|
assert.equal(parseArguments([...base, ...audience])['admin-audience'], 'https://relay.example.test')
|
|
assert.throws(
|
|
() => parseArguments([...base, '--expected-rehome-generation', '1', '--rehome-control-origin', 'https://relay.example.test']),
|
|
/durable rehome verification arguments must be configured together/
|
|
)
|
|
assert.throws(
|
|
() => parseArguments([...base, ...audience, '--expected-rehome-generation', '1']),
|
|
/durable rehome verification arguments must be configured together/
|
|
)
|
|
})
|
|
|
|
test('--shadow-seat-feed-cells sets the director setting, and preserve keeps the serving one', async () => {
|
|
assert.equal(
|
|
directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'c1, c2' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV],
|
|
'c1,c2'
|
|
)
|
|
assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'all' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'all')
|
|
assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'preserve' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], undefined)
|
|
assert.throws(() => directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'C1;x' }), /must be preserve, all, or a comma list/)
|
|
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
|
const harness = directorHarness({ servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' } })
|
|
await deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'preserve' }, 'candidate-new', harness.operations)
|
|
assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'c1,c3')
|
|
const shrink = directorHarness({
|
|
servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' },
|
|
reserveCells: { 'https://relay-hash-uc.a.run.app': ['c3'], 'https://candidate-new---relay-hash-uc.a.run.app': ['c3'] }
|
|
})
|
|
await assert.rejects(
|
|
deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'c1' }, 'candidate-new', shrink.operations),
|
|
/would drop reserve cells \(c3\)/
|
|
)
|
|
})
|