Files
orca/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
T
Jinwoo Hong 2524737ef0 chore(relay): apply the cell backend drain and request-logging settings inside each same-cap wave (#21860)
* chore(relay): target each cell's backend service from the same-cap job

The 60 s connection drain timeout merged in #21848 has no safe apply path.
A root plan scoped to the backend services alone still pulls every
`google_compute_instance_template.relay_gce_cell` in as a dependency, and
standing image drift turns all 29 into replacements, so applying it would roll
the fleet at once.

Add `google_compute_backend_service.relay_gce_cell["${TARGET_CELL_ID}"]` to
both plan invocations in the per-cell same-cap job, next to the template and
MIG it already targets, and teach the reviewed plan validator to allow exactly
one extra change: an in-place update of that one cell's backend whose only
changed attribute is `connection_draining_timeout_sec`, landing on the
constant `validate-relay-asia-topology-plan.mjs` exports. Any other attribute,
any other resource, or a backend for another cell still fails the validator.

The accepted update is reported as `connectionDrainUpdate` and kept out of
`changes`, so the apply step's stranded branch and the resume step's drift
branch keep reading the template-and-MIG count they were written against; the
resume branch additionally accepts a plan whose only pending change is that
drain update, which restarts nothing.

Claude-Session: relay-same-cap-targets-cell-backend

* fix(relay): also let the same-cap wave apply this cell's LB request logging

A read-only production plan for production-gce-c7 showed the live US cell
backends carry no `log_config` at all, while relay-gce-cells.tf has declared
`log_config { enable = true, sample_rate = var.relay_gce_cell_log_sample_rate }`
on every cell backend since the Terraform root landed in 3eec77c11a (#18413).
Nothing has applied it because every production apply since is a per-cell
targeted plan that names only the template and the MIG.

So the real canary plan's backend moves two paths, not one:
`["connection_draining_timeout_sec", "log_config.0"]`. The drain-only validator
rejected exactly that plan, which would have stranded the cell mid-wave after
the drain had already started.

Accept both, each optional, for this cell's backend only: the drain landing on
RELAY_CELL_CONNECTION_DRAIN_SECONDS, and a log_config of exactly one block with
`enable = true` and `sample_rate` equal to RELAY_CELL_LOG_SAMPLE_RATE, the
declared default of a variable no environment file overrides. Any third path,
a different sample rate, disabled logging, another cell, or a replacement still
fails. The accepted paths are reported as `backendUpdate`, which the resume
branch now reads instead of the drain-only flag.

Verified against the real production plan: the masked, three-target plan for
production-gce-c7 contains exactly that cell's template, MIG, and backend
service and nothing else, and this validator returns
`{"changes":2,"backendUpdate":["connection_draining_timeout_sec","log_config.0"]}`.

Claude-Session: relay-same-cap-targets-cell-backend
2026-09-20 20:35:52 -04:00

1229 lines
44 KiB
JavaScript

import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
RELAY_CELL_LOG_SAMPLE_RATE
} from './validate-relay-asia-topology-plan.mjs'
import {
parseCapacityPlanArguments,
validateCapacityPlan as validateCapacityPlanRaw
} from './validate-relay-capacity-plan.mjs'
const config = {
cellId: 'staging-gce-c3',
hardCap: 1_000,
unobservedBound: 60
}
function replacementConfiguration(references = [
'google_compute_instance_template.relay_gce_cell',
'each.key'
]) {
return {
root_module: {
resources: [{
address: 'google_compute_instance_group_manager.relay_gce_cell',
expressions: {
version: [{
instance_template: { references },
name: { constant_value: 'primary' }
}]
}
}]
}
}
}
function validateCapacityPlan(plan, planConfig) {
const replacement = plan.resource_changes.some(({ change }) =>
JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']))
return validateCapacityPlanRaw(
replacement && !plan.configuration
? { ...plan, configuration: replacementConfiguration() }
: plan,
planConfig
)
}
test('accepts only the exact canary template replacement and MIG update', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}`
const startupScript = (cap, bound, selectedImage, extra = '') =>
[
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
extra,
'docker run --detach \\',
' --name cloud-sql-proxy \\',
` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const script = startupScript(1_000, 60, image)
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startupScript(
600,
60,
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
)
},
after: { metadata_startup_script: script, self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const cellConfig = { ...config, mode: 'cell', image }
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), {
mode: 'cell',
changes: 0
})
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
const wrongTemplateManager = structuredClone(manager)
wrongTemplateManager.change.after.version[0].instance_template =
'projects/project/global/instanceTemplates/unreviewed'
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, wrongTemplateManager] },
cellConfig
),
/does not bind the MIG/
)
assert.throws(
() => validateCapacityPlanRaw(
{
resource_changes: [template, manager],
configuration: replacementConfiguration([
'google_compute_instance_template.relay_gce_cell',
'var.unreviewed_key'
])
},
cellConfig
),
/reviewed template dependency/
)
assert.throws(
() => validateCapacityPlanRaw(
{ resource_changes: [template, manager] },
cellConfig
),
/reviewed template dependency/
)
const capacityServiceAccount =
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
const bootstrapTemplate = structuredClone(template)
const bootstrapManager = structuredClone(manager)
bootstrapTemplate.change.after.metadata_startup_script = [
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`,
script
].join('\n')
const bootstrapConfig = {
...cellConfig,
mode: 'bootstrap-cell',
capacityServiceAccount
}
const plannedValues = (startupScript) => ({
root_module: {
resources: [
{
address: bootstrapTemplate.address,
values: {
metadata_startup_script: startupScript,
self_link: 'projects/project/global/instanceTemplates/c26-reviewed'
}
},
{
address: bootstrapManager.address,
values: {
version: [{
instance_template:
'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed'
}]
}
}
]
}
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
const managerOnly = structuredClone(bootstrapManager)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [managerOnly],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 1 }
)
const decoyPlannedValues = plannedValues(
bootstrapTemplate.change.after.metadata_startup_script.replaceAll(
image,
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
) + `\n# decoy '${image}'`
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly], planned_values: decoyPlannedValues },
bootstrapConfig
),
/reviewed image and capacity/
)
const obsoleteTemplate = structuredClone(bootstrapTemplate)
obsoleteTemplate.deposed = 'retired-template'
obsoleteTemplate.change.actions = ['delete']
obsoleteTemplate.change.after = null
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [managerOnly, obsoleteTemplate],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [obsoleteTemplate],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 1 }
)
const wrongManagerReference = plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
wrongManagerReference.root_module.resources[1].values.version[0].instance_template =
'projects/project/global/instanceTemplates/not-reviewed'
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly], planned_values: wrongManagerReference },
bootstrapConfig
),
/does not bind the MIG/
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly] },
bootstrapConfig
),
/no exact planned C26 state/
)
const restartedBootstrapManager = structuredClone(bootstrapManager)
restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }]
restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }]
restartedBootstrapManager.change.before.version[0].name =
'0/2026-08-10 23:30:14.196895+00:00'
restartedBootstrapManager.change.after.version[0].name = 'primary'
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, restartedBootstrapManager] },
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [template, restartedBootstrapManager] },
cellConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedRestartManager = structuredClone(restartedBootstrapManager)
unrecognizedRestartManager.change.before.version[0].name = 'operator-version'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedRestartManager] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager)
unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager)
unrecognizedPrimaryVersion.change.after.version[0].name = 'other'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager)
unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const missingRestartPolicy = structuredClone(restartedBootstrapManager)
delete missingRestartPolicy.change.before.update_policy
delete missingRestartPolicy.change.after.update_policy
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, missingRestartPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const missingRestartVersion = structuredClone(restartedBootstrapManager)
missingRestartVersion.change.before.version[0].name = 'primary'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, missingRestartVersion] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const duplicateHardCapTemplate = structuredClone(template)
duplicateHardCapTemplate.change.after.metadata_startup_script = [
script,
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`
].join('\n')
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] },
cellConfig
),
/reviewed image and capacity/
)
const duplicateIdentityTemplate = structuredClone(bootstrapTemplate)
duplicateIdentityTemplate.change.after.metadata_startup_script = [
duplicateIdentityTemplate.change.after.metadata_startup_script,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'`
].join('\n')
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] },
bootstrapConfig
),
/reviewed image and capacity/
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
{ ...bootstrapConfig, capacityServiceAccount: 'invalid' }
),
/invalid service account/
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
cellConfig
),
/reviewed image and capacity/
)
manager.change.after.target_size = 0
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
manager.change.after.target_size = 1
template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad')
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/reviewed image and capacity/
)
template.change.after.metadata_startup_script = startupScript(
1_000,
60,
image,
'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP='
)
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/reviewed image and capacity/
)
template.change.after.metadata_startup_script = script
template.change.after_unknown = {
id: true,
self_link: true,
disk: [{ architecture: true }]
}
manager.change.after_unknown = {
fingerprint: true,
version: [{ instance_template: true }]
}
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
template.change.before.description = ''
template.change.after.description = null
template.change.before.disk = [{
architecture: '',
source_image: 'projects/cos-cloud/global/images/cos-stable-1'
}]
template.change.after.disk = [{
source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
}]
template.change.after_unknown.disk = [{ architecture: true }]
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
template.change.after.disk[0].source_image =
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different'
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
template.change.after.disk[0].source_image =
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
manager.change.after_unknown = { target_size: true }
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
})
test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
after: {
metadata_startup_script: startup({ selectedImage: image, trust: true }),
self_link: null
},
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const sameCapConfig = {
...config,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
// A pre-template-apply rollback resume validates drift for the image the
// cell already serves: the template leaves and re-enters the rollback image.
const resumeTemplate = structuredClone(template)
resumeTemplate.change.after.metadata_startup_script = startup({
selectedImage: rollbackImage,
trust: true
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [resumeTemplate, manager] },
{ ...sameCapConfig, image: rollbackImage }
),
{ mode: 'same-cap-cell', changes: 2 }
)
const asiaTemplate = structuredClone(template)
const asiaManager = structuredClone(manager)
asiaTemplate.address =
'google_compute_instance_template.relay_gce_cell["production-gce-c28"]'
asiaManager.address =
'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]'
asiaTemplate.change.before.metadata_startup_script = startup({
selectedImage: rollbackImage,
cap: 3_000
})
asiaTemplate.change.after.metadata_startup_script = startup({
selectedImage: image,
cap: 3_000,
trust: true
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [asiaTemplate, asiaManager] },
{ ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 }
),
{ mode: 'same-cap-cell', changes: 2 }
)
const changedCap = structuredClone(template)
changedCap.change.before.metadata_startup_script = startup({
selectedImage: rollbackImage,
cap: 600
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig),
/reviewed image and capacity/
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...sameCapConfig, rollbackImage: image }
),
/reviewed image and capacity/
)
const wrongTrust = structuredClone(template)
wrongTrust.change.after.metadata_startup_script = startup({
selectedImage: image,
trust: true
}).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com')
assert.throws(
() => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig),
/reviewed image and capacity/
)
const imageOnly = structuredClone(template)
imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image })
const imageOnlyConfig = {
...config,
mode: 'same-cap-image',
image,
rollbackImage
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig),
{ mode: 'same-cap-image', changes: 2, changeKind: 'replacement' }
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [imageOnly, manager] },
{ ...imageOnlyConfig, rollbackImage: image }
),
/reviewed image and capacity/
)
const changedTrust = structuredClone(imageOnly)
changedTrust.change.after.metadata_startup_script +=
`\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
assert.throws(
() => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig),
/reviewed image and capacity/
)
const plannedValues = {
root_module: {
resources: [
{
address: imageOnly.address,
values: {
metadata_startup_script: imageOnly.change.after.metadata_startup_script,
self_link: 'projects/project/global/instanceTemplates/target'
}
},
{
address: manager.address,
values: {
version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }]
}
}
]
}
}
const obsoleteTemplate = structuredClone(imageOnly)
obsoleteTemplate.deposed = 'obsolete'
obsoleteTemplate.change.actions = ['delete']
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [obsoleteTemplate], planned_values: plannedValues },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' }
)
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate] },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' }
)
const anotherObsoleteTemplate = {
...structuredClone(obsoleteTemplate),
deposed: 'another-obsolete'
}
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [obsoleteTemplate, anotherObsoleteTemplate],
planned_values: plannedValues
},
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate],
planned_values: plannedValues
},
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' }
)
const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate)
invalidObsoleteTemplate.change.actions = ['update']
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] },
imageOnlyConfig
),
/change only the exact instance template and MIG/
)
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [manager], planned_values: plannedValues },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' }
)
})
test('protocol-0 same-cap cells roll without rehome trust lines', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, trust = false }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
const gainsTrust = structuredClone(template)
gainsTrust.change.after.metadata_startup_script = startup({
selectedImage: image,
trust: true
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig),
/reviewed image and capacity/
)
// Under protocol 1 that same script is the reviewed roll: trust is added, not drift.
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [gainsTrust, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
{ mode: 'same-cap-cell', changes: 2 }
)
// A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged.
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
/reviewed image and capacity/
)
for (const protocol of [undefined, '', '2', 'yes']) {
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: protocol }
),
/invalid regional rehome protocol/
)
}
})
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, capacity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const plan = (beforeCapacity, afterCapacity) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({
selectedImage: rollbackImage,
capacity: beforeCapacity
})
},
after: {
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const config = {
cellId: 'production-gce-c17',
hardCap: 600,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
// A template old enough to predate the line gains it, which is the only move allowed.
assert.deepEqual(
validateCapacityPlan(plan(null, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
for (const [before, after] of [
[capacityIdentity, null],
[null, null],
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
]) {
assert.throws(
() => validateCapacityPlan(plan(before, after), config),
/reviewed image and capacity/,
`${before} -> ${after}`
)
}
// Without the pin there is nothing reviewing the line the comparison now ignores.
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: undefined
}),
/invalid service account/
)
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: 'not-an-email'
}),
/invalid service account/
)
})
test('the capacity identity argument is required by same-cap-cell mode', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const base = [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c17',
'--hard-cap', '600',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '0'
]
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
assert.throws(
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
/--capacity-service-account is invalid/
)
assert.equal(
parseCapacityPlanArguments([
...base,
'--capacity-service-account',
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
]).capacityServiceAccount,
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
)
})
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0'))
.regionalRehomeProtocol,
'0'
)
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
'3'
)
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments()),
/requires rollback image and rehome trust config/
)
for (const protocol of ['', '2', 'true']) {
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)),
/requires rollback image and rehome trust config/
)
}
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--regional-rehome-protocol', '0'
]),
/applies only to same-cap-cell validation/
)
})
test('the reviewed database pool is pinned for the cells that emit one', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, pool }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const rollPlan = (before, after) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
},
after: {
metadata_startup_script: startup({ selectedImage: image, pool: after }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
assert.deepEqual(
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
/reviewed image and capacity/
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
// A cell on the root pool default emits no line at all, and gaining one is real drift.
assert.deepEqual(
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
/reviewed image and capacity/
)
// A line already on the live template is still unreviewed without the pin, even standing still.
assert.throws(
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
/reviewed image and capacity/
)
// A pin must also fail closed when the plan drops the line it names.
assert.throws(
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
for (const pool of ['', '0', '101', 'ten']) {
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
/invalid database pool max/
)
}
})
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '1',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
'16'
)
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--database-pool-max', '16'
]),
/applies only to same-cap-cell validation/
)
})
test('a same-cap roll may carry only this cell backend drain and request logging', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const startup = (selectedImage) => [
" printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '1000'",
" printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'",
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup(rollbackImage) },
after: { metadata_startup_script: startup(image), self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
// The exact shape a live US cell's backend plans: 300 s drain and no log_config at all.
const loggingAfter = [{
enable: true,
optional_fields: null,
optional_mode: null,
sample_rate: RELAY_CELL_LOG_SAMPLE_RATE
}]
const backendChange = ({ drain = true, logging = true }) => ({
address: 'google_compute_backend_service.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: {
connection_draining_timeout_sec: drain ? 300 : RELAY_CELL_CONNECTION_DRAIN_SECONDS,
log_config: logging ? [] : loggingAfter,
timeout_sec: 86_400,
fingerprint: 'before'
},
after: {
connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS,
log_config: loggingAfter,
timeout_sec: 86_400,
fingerprint: null
},
after_unknown: { fingerprint: true }
}
})
const backend = backendChange({})
const drainOnly = backendChange({ logging: false })
const loggingOnly = backendChange({ drain: false })
const sameCapConfig = {
...config,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: 'relay-director@project.iam.gserviceaccount.com',
rehomeAudience: 'https://relay.onorca.dev/v1/admin/host-drain',
regionalRehomeProtocol: '0'
}
const refused = /only this cell backend drain and request logging/
// Both attributes ride along with the roll without inflating the template-and-MIG count
// the apply's stranded branch and the resume's drift branch both read.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, backend] }, sameCapConfig),
{
mode: 'same-cap-cell',
changes: 2,
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
}
)
// Each is independently optional: a cell that already has one plans no change for it.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, drainOnly] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['connection_draining_timeout_sec'] }
)
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, loggingOnly] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['log_config.0'] }
)
// Once both are applied the backend is simply absent from the plan.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
// A cell whose template and MIG have converged but whose backend has not is still clean.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [backend] }, sameCapConfig),
{
mode: 'same-cap-cell',
changes: 0,
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
}
)
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, sameCapConfig), {
mode: 'same-cap-cell',
changes: 0
})
const extraAttribute = structuredClone(backend)
extraAttribute.change.after.timeout_sec = 3_600
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, extraAttribute] },
sameCapConfig
),
/changes outside the reviewed capacity fields/
)
const otherCell = structuredClone(backend)
otherCell.address = 'google_compute_backend_service.relay_gce_cell["production-gce-c27"]'
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager, otherCell] }, sameCapConfig),
refused
)
const unreviewedDrain = structuredClone(backend)
unreviewedDrain.change.after.connection_draining_timeout_sec =
RELAY_CELL_CONNECTION_DRAIN_SECONDS + 1
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, unreviewedDrain] },
sameCapConfig
),
refused
)
const sampledLogging = structuredClone(backend)
sampledLogging.change.after.log_config[0].sample_rate = RELAY_CELL_LOG_SAMPLE_RATE / 2
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, sampledLogging] },
sameCapConfig
),
refused
)
const disabledLogging = structuredClone(backend)
disabledLogging.change.after.log_config[0].enable = false
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, disabledLogging] },
sameCapConfig
),
refused
)
const replaced = structuredClone(backend)
replaced.change.actions = ['create', 'delete']
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager, replaced] }, sameCapConfig),
refused
)
// Only the same-cap wave targets a backend service; every other mode still refuses one.
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, backend] },
{ ...config, mode: 'cell', image }
),
/only the exact instance template and MIG/
)
})