Files
orca/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
T
Jinwoo Hong 744e7722c2 fix(relay): accept MIG version-name reconciliation and recreate stranded cells without rewriting the MIG (#24373)
* fix(relay): accept MIG version-name reconciliation and recreate stranded cells without rewriting the MIG

The stranded-rollback recovery ran a gcloud rolling action, which renames the
MIG version outside Terraform. Every later plan for that cell then reverted the
label, and the capacity-plan validator refused the revert as an unreviewed MIG
change, so the cell could be neither rolled nor rolled back.

The validator now accepts a MIG field moving back to what relay-gce-cells.tf
declares (version name and update policy), in every mode, and a test pins those
values to the Terraform file. The stranded branch recreates the cell's single
instance with recreate-instances, which leaves the MIG untouched.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* fix(relay): let a label-only MIG plan through and recreate on it in a stranded rollback

A stranded rollback whose template is already in place plans only the version
name revert. The validator still required the MIG template to move, so that
plan was refused, and the recreate gate (changes == 0) would have skipped a
plan of one change and left the drain flag set. Require the template move only
when no declared field reconciles, and recreate whenever the template was not
replaced (changes < 2).

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
2026-10-01 08:12:49 -04:00

1431 lines
52 KiB
JavaScript

import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
RELAY_CELL_LOG_SAMPLE_RATE
} from './validate-relay-asia-topology-plan.mjs'
import { readFileSync } from 'node:fs'
import {
DECLARED_MANAGER_FIELDS,
parseCapacityPlanArguments,
validateCapacityPlan as validateCapacityPlanRaw
} from './validate-relay-capacity-plan.mjs'
const config = {
cellId: 'staging-gce-c3',
hardCap: 1_000,
unobservedBound: 60
}
function replacementConfiguration(references = [
'google_compute_instance_template.relay_gce_cell',
'each.key'
]) {
return {
root_module: {
resources: [{
address: 'google_compute_instance_group_manager.relay_gce_cell',
expressions: {
version: [{
instance_template: { references },
name: { constant_value: 'primary' }
}]
}
}]
}
}
}
function validateCapacityPlan(plan, planConfig) {
const replacement = plan.resource_changes.some(({ change }) =>
JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']))
return validateCapacityPlanRaw(
replacement && !plan.configuration
? { ...plan, configuration: replacementConfiguration() }
: plan,
planConfig
)
}
test('accepts only the exact canary template replacement and MIG update', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}`
const startupScript = (cap, bound, selectedImage, extra = '') =>
[
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
extra,
'docker run --detach \\',
' --name cloud-sql-proxy \\',
` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const script = startupScript(1_000, 60, image)
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startupScript(
600,
60,
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
)
},
after: { metadata_startup_script: script, self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const cellConfig = { ...config, mode: 'cell', image }
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), {
mode: 'cell',
changes: 0
})
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
const wrongTemplateManager = structuredClone(manager)
wrongTemplateManager.change.after.version[0].instance_template =
'projects/project/global/instanceTemplates/unreviewed'
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, wrongTemplateManager] },
cellConfig
),
/does not bind the MIG/
)
assert.throws(
() => validateCapacityPlanRaw(
{
resource_changes: [template, manager],
configuration: replacementConfiguration([
'google_compute_instance_template.relay_gce_cell',
'var.unreviewed_key'
])
},
cellConfig
),
/reviewed template dependency/
)
assert.throws(
() => validateCapacityPlanRaw(
{ resource_changes: [template, manager] },
cellConfig
),
/reviewed template dependency/
)
const capacityServiceAccount =
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
const bootstrapTemplate = structuredClone(template)
const bootstrapManager = structuredClone(manager)
bootstrapTemplate.change.after.metadata_startup_script = [
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`,
script
].join('\n')
const bootstrapConfig = {
...cellConfig,
mode: 'bootstrap-cell',
capacityServiceAccount
}
const plannedValues = (startupScript) => ({
root_module: {
resources: [
{
address: bootstrapTemplate.address,
values: {
metadata_startup_script: startupScript,
self_link: 'projects/project/global/instanceTemplates/c26-reviewed'
}
},
{
address: bootstrapManager.address,
values: {
version: [{
instance_template:
'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed'
}]
}
}
]
}
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
const managerOnly = structuredClone(bootstrapManager)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [managerOnly],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 1 }
)
const decoyPlannedValues = plannedValues(
bootstrapTemplate.change.after.metadata_startup_script.replaceAll(
image,
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
) + `\n# decoy '${image}'`
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly], planned_values: decoyPlannedValues },
bootstrapConfig
),
/reviewed image and capacity/
)
const obsoleteTemplate = structuredClone(bootstrapTemplate)
obsoleteTemplate.deposed = 'retired-template'
obsoleteTemplate.change.actions = ['delete']
obsoleteTemplate.change.after = null
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [managerOnly, obsoleteTemplate],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [obsoleteTemplate],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 1 }
)
const wrongManagerReference = plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
wrongManagerReference.root_module.resources[1].values.version[0].instance_template =
'projects/project/global/instanceTemplates/not-reviewed'
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly], planned_values: wrongManagerReference },
bootstrapConfig
),
/does not bind the MIG/
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly] },
bootstrapConfig
),
/no exact planned C26 state/
)
// A MIG moving back to its declared label and update policy is reconciliation in every mode.
const restartedBootstrapManager = structuredClone(bootstrapManager)
restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }]
restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }]
restartedBootstrapManager.change.before.version[0].name =
'0/2026-08-10 23:30:14.196895+00:00'
restartedBootstrapManager.change.after.version[0].name = 'primary'
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, restartedBootstrapManager] },
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, restartedBootstrapManager] }, cellConfig),
{ mode: 'cell', changes: 2 }
)
const operatorLabelManager = structuredClone(restartedBootstrapManager)
operatorLabelManager.change.before.version[0].name = 'operator-version'
delete operatorLabelManager.change.before.update_policy
delete operatorLabelManager.change.after.update_policy
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, operatorLabelManager] },
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager)
unrecognizedPrimaryVersion.change.after.version[0].name = 'other'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager)
unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
// Gaining a whole policy block is not a field reverting, so it is not reconciliation.
const addedPolicy = structuredClone(restartedBootstrapManager)
delete addedPolicy.change.before.update_policy
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, addedPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const duplicateHardCapTemplate = structuredClone(template)
duplicateHardCapTemplate.change.after.metadata_startup_script = [
script,
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`
].join('\n')
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] },
cellConfig
),
/reviewed image and capacity/
)
const duplicateIdentityTemplate = structuredClone(bootstrapTemplate)
duplicateIdentityTemplate.change.after.metadata_startup_script = [
duplicateIdentityTemplate.change.after.metadata_startup_script,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'`
].join('\n')
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] },
bootstrapConfig
),
/reviewed image and capacity/
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
{ ...bootstrapConfig, capacityServiceAccount: 'invalid' }
),
/invalid service account/
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
cellConfig
),
/reviewed image and capacity/
)
manager.change.after.target_size = 0
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
manager.change.after.target_size = 1
template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad')
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/reviewed image and capacity/
)
template.change.after.metadata_startup_script = startupScript(
1_000,
60,
image,
'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP='
)
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/reviewed image and capacity/
)
template.change.after.metadata_startup_script = script
template.change.after_unknown = {
id: true,
self_link: true,
disk: [{ architecture: true }]
}
manager.change.after_unknown = {
fingerprint: true,
version: [{ instance_template: true }]
}
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
template.change.before.description = ''
template.change.after.description = null
template.change.before.disk = [{
architecture: '',
source_image: 'projects/cos-cloud/global/images/cos-stable-1'
}]
template.change.after.disk = [{
source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
}]
template.change.after_unknown.disk = [{ architecture: true }]
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
template.change.after.disk[0].source_image =
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different'
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
template.change.after.disk[0].source_image =
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
manager.change.after_unknown = { target_size: true }
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
})
test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
after: {
metadata_startup_script: startup({ selectedImage: image, trust: true }),
self_link: null
},
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const sameCapConfig = {
...config,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
// A gcloud rolling action on a stranded cell renamed its MIG version; the next roll reverts it.
const relabelledManager = structuredClone(manager)
relabelledManager.change.before.version[0].name = '0/2026-10-01 10:41:28.681518+00:00'
relabelledManager.change.after.version[0].name = 'primary'
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, relabelledManager] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
for (const [field, moved] of [
['target_size', 0],
['base_instance_name', 'relay-other'],
['named_port', [{ name: 'relay', port: 9090 }]]
]) {
const unrelated = structuredClone(relabelledManager)
unrelated.change.after[field] = moved
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, unrelated] }, sameCapConfig),
/outside the reviewed capacity fields/,
field
)
}
const declaredPolicy = {
type: 'PROACTIVE',
minimal_action: 'REPLACE',
most_disruptive_allowed_action: 'REPLACE',
replacement_method: 'RECREATE',
max_surge_fixed: 0,
max_unavailable_fixed: 1
}
for (const [field, drifted, away] of [
['type', 'OPPORTUNISTIC', 'OPPORTUNISTIC'],
['minimal_action', 'RESTART', 'RESTART'],
['most_disruptive_allowed_action', 'RESTART', 'RESTART'],
['replacement_method', 'SUBSTITUTE', 'SUBSTITUTE'],
['max_surge_fixed', 1, 1],
['max_unavailable_fixed', 0, 0]
]) {
const reverted = structuredClone(relabelledManager)
reverted.change.before.update_policy = [{ ...declaredPolicy, [field]: drifted }]
reverted.change.after.update_policy = [{ ...declaredPolicy }]
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, reverted] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 },
field
)
const leaving = structuredClone(relabelledManager)
leaving.change.before.update_policy = [{ ...declaredPolicy }]
leaving.change.after.update_policy = [{ ...declaredPolicy, [field]: away }]
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, leaving] }, sameCapConfig),
/outside the reviewed capacity fields/,
field
)
}
// A second version is a canary split, never a label revert.
const secondVersion = structuredClone(relabelledManager)
secondVersion.change.after.version.push({
name: 'primary',
instance_template: 'projects/project/global/instanceTemplates/canary',
target_size: [{ fixed: 1 }]
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, secondVersion] }, sameCapConfig),
/outside the reviewed capacity fields/
)
// A stranded rollback whose template is already in place plans only the label revert.
const plannedCell = (startupScript, templateLink) => ({
root_module: {
resources: [
{
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
values: { metadata_startup_script: startupScript, self_link: templateLink }
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
values: { version: [{ instance_template: templateLink, name: 'primary' }] }
}
]
}
})
const reviewedLink = 'projects/project/global/instanceTemplates/reviewed'
const labelOnly = {
address: manager.address,
change: {
actions: ['update'],
before: {
target_size: 1,
version: [{ instance_template: reviewedLink, name: '0/2026-10-01 10:41:28.681518+00:00' }]
},
after: { target_size: 1, version: [{ instance_template: reviewedLink, name: 'primary' }] },
after_unknown: {}
}
}
const strandedPlan = (managerChange) => ({
resource_changes: [managerChange],
planned_values: plannedCell(template.change.after.metadata_startup_script, reviewedLink)
})
assert.deepEqual(
validateCapacityPlan(strandedPlan(labelOnly), sameCapConfig),
{ mode: 'same-cap-cell', changes: 1 }
)
const labelAway = structuredClone(labelOnly)
labelAway.change.before.version[0].name = 'primary'
labelAway.change.after.version[0].name = 'other'
assert.throws(
() => validateCapacityPlan(strandedPlan(labelAway), sameCapConfig),
/outside the reviewed capacity fields/
)
const labelAndResize = structuredClone(labelOnly)
labelAndResize.change.after.target_size = 0
assert.throws(
() => validateCapacityPlan(strandedPlan(labelAndResize), sameCapConfig),
/outside the reviewed capacity fields/
)
const percentSurge = structuredClone(relabelledManager)
percentSurge.change.before.update_policy = [{ ...declaredPolicy, max_surge_percent: 0 }]
percentSurge.change.after.update_policy = [{ ...declaredPolicy, max_surge_percent: 50 }]
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, percentSurge] }, sameCapConfig),
/outside the reviewed capacity fields/
)
// A pre-template-apply rollback resume validates drift for the image the
// cell already serves: the template leaves and re-enters the rollback image.
const resumeTemplate = structuredClone(template)
resumeTemplate.change.after.metadata_startup_script = startup({
selectedImage: rollbackImage,
trust: true
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [resumeTemplate, manager] },
{ ...sameCapConfig, image: rollbackImage }
),
{ mode: 'same-cap-cell', changes: 2 }
)
const asiaTemplate = structuredClone(template)
const asiaManager = structuredClone(manager)
asiaTemplate.address =
'google_compute_instance_template.relay_gce_cell["production-gce-c28"]'
asiaManager.address =
'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]'
asiaTemplate.change.before.metadata_startup_script = startup({
selectedImage: rollbackImage,
cap: 3_000
})
asiaTemplate.change.after.metadata_startup_script = startup({
selectedImage: image,
cap: 3_000,
trust: true
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [asiaTemplate, asiaManager] },
{ ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 }
),
{ mode: 'same-cap-cell', changes: 2 }
)
const changedCap = structuredClone(template)
changedCap.change.before.metadata_startup_script = startup({
selectedImage: rollbackImage,
cap: 600
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig),
/reviewed image and capacity/
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...sameCapConfig, rollbackImage: image }
),
/reviewed image and capacity/
)
const wrongTrust = structuredClone(template)
wrongTrust.change.after.metadata_startup_script = startup({
selectedImage: image,
trust: true
}).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com')
assert.throws(
() => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig),
/reviewed image and capacity/
)
// Production run 35684694704 died after create_before_destroy made the new template, leaving
// the Sep 18 template deposed; every later wave plan for that cell carries its delete.
const deposedTemplate = structuredClone(template)
deposedTemplate.deposed = '4cb19f83'
deposedTemplate.change.actions = ['delete']
deposedTemplate.change.after = null
deposedTemplate.change.after_unknown = {}
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [template, manager, deposedTemplate] },
sameCapConfig
),
{ mode: 'same-cap-cell', changes: 2, obsoleteTemplates: 1 }
)
const updatedDeposedTemplate = structuredClone(deposedTemplate)
updatedDeposedTemplate.change.actions = ['update']
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, updatedDeposedTemplate] },
sameCapConfig
),
/change only the exact instance template and MIG/
)
const foreignDeposedTemplate = structuredClone(deposedTemplate)
foreignDeposedTemplate.address =
'google_compute_instance_template.relay_gce_cell["production-gce-c8"]'
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, foreignDeposedTemplate] },
sameCapConfig
),
/change only the exact instance template and MIG/
)
const secondDeposedTemplate = { ...structuredClone(deposedTemplate), deposed: 'aa17b204' }
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, deposedTemplate, secondDeposedTemplate] },
sameCapConfig
),
/change only the exact instance template and MIG/
)
// Nothing but the deposed delete left: the cell is stranded on the reviewed template, and the
// job's `changes == 0` roll must still fire.
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [deposedTemplate],
planned_values: {
root_module: {
resources: [
{
address: template.address,
values: {
metadata_startup_script: template.change.after.metadata_startup_script,
self_link: 'projects/project/global/instanceTemplates/target'
}
},
{
address: manager.address,
values: {
version: [{
instance_template: 'projects/project/global/instanceTemplates/target'
}]
}
}
]
}
}
},
sameCapConfig
),
{ mode: 'same-cap-cell', changes: 0, obsoleteTemplates: 1 }
)
const imageOnly = structuredClone(template)
imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image })
const imageOnlyConfig = {
...config,
mode: 'same-cap-image',
image,
rollbackImage
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig),
{ mode: 'same-cap-image', changes: 2, changeKind: 'replacement' }
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [imageOnly, manager] },
{ ...imageOnlyConfig, rollbackImage: image }
),
/reviewed image and capacity/
)
const changedTrust = structuredClone(imageOnly)
changedTrust.change.after.metadata_startup_script +=
`\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
assert.throws(
() => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig),
/reviewed image and capacity/
)
const plannedValues = {
root_module: {
resources: [
{
address: imageOnly.address,
values: {
metadata_startup_script: imageOnly.change.after.metadata_startup_script,
self_link: 'projects/project/global/instanceTemplates/target'
}
},
{
address: manager.address,
values: {
version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }]
}
}
]
}
}
const obsoleteTemplate = structuredClone(imageOnly)
obsoleteTemplate.deposed = 'obsolete'
obsoleteTemplate.change.actions = ['delete']
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [obsoleteTemplate], planned_values: plannedValues },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' }
)
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate] },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' }
)
const anotherObsoleteTemplate = {
...structuredClone(obsoleteTemplate),
deposed: 'another-obsolete'
}
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [obsoleteTemplate, anotherObsoleteTemplate],
planned_values: plannedValues
},
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate],
planned_values: plannedValues
},
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' }
)
const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate)
invalidObsoleteTemplate.change.actions = ['update']
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] },
imageOnlyConfig
),
/change only the exact instance template and MIG/
)
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [manager], planned_values: plannedValues },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' }
)
})
test('protocol-0 same-cap cells roll without rehome trust lines', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, trust = false }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
const gainsTrust = structuredClone(template)
gainsTrust.change.after.metadata_startup_script = startup({
selectedImage: image,
trust: true
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig),
/reviewed image and capacity/
)
// Under protocol 1 that same script is the reviewed roll: trust is added, not drift.
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [gainsTrust, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
{ mode: 'same-cap-cell', changes: 2 }
)
// A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged.
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
/reviewed image and capacity/
)
for (const protocol of [undefined, '', '2', 'yes']) {
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: protocol }
),
/invalid regional rehome protocol/
)
}
})
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, capacity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const plan = (beforeCapacity, afterCapacity) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({
selectedImage: rollbackImage,
capacity: beforeCapacity
})
},
after: {
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const config = {
cellId: 'production-gce-c17',
hardCap: 600,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
// A template old enough to predate the line gains it, which is the only move allowed.
assert.deepEqual(
validateCapacityPlan(plan(null, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
for (const [before, after] of [
[capacityIdentity, null],
[null, null],
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
]) {
assert.throws(
() => validateCapacityPlan(plan(before, after), config),
/reviewed image and capacity/,
`${before} -> ${after}`
)
}
// Without the pin there is nothing reviewing the line the comparison now ignores.
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: undefined
}),
/invalid service account/
)
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: 'not-an-email'
}),
/invalid service account/
)
})
test('the capacity identity argument is required by same-cap-cell mode', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const base = [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c17',
'--hard-cap', '600',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '0'
]
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
assert.throws(
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
/--capacity-service-account is invalid/
)
assert.equal(
parseCapacityPlanArguments([
...base,
'--capacity-service-account',
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
]).capacityServiceAccount,
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
)
})
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0'))
.regionalRehomeProtocol,
'0'
)
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
'3'
)
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments()),
/requires rollback image and rehome trust config/
)
for (const protocol of ['', '2', 'true']) {
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)),
/requires rollback image and rehome trust config/
)
}
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--regional-rehome-protocol', '0'
]),
/applies only to same-cap-cell validation/
)
})
test('the reviewed database pool is pinned for the cells that emit one', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, pool }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const rollPlan = (before, after) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
},
after: {
metadata_startup_script: startup({ selectedImage: image, pool: after }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
assert.deepEqual(
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
/reviewed image and capacity/
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
// A cell on the root pool default emits no line at all, and gaining one is real drift.
assert.deepEqual(
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
/reviewed image and capacity/
)
// A line already on the live template is still unreviewed without the pin, even standing still.
assert.throws(
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
/reviewed image and capacity/
)
// A pin must also fail closed when the plan drops the line it names.
assert.throws(
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
for (const pool of ['', '0', '101', 'ten']) {
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
/invalid database pool max/
)
}
})
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '1',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
'16'
)
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--database-pool-max', '16'
]),
/applies only to same-cap-cell validation/
)
})
test('a same-cap roll may carry only this cell backend drain and request logging', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const startup = (selectedImage) => [
" printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '1000'",
" printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'",
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup(rollbackImage) },
after: { metadata_startup_script: startup(image), self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
// The exact shape a live US cell's backend plans: 300 s drain and no log_config at all.
const loggingAfter = [{
enable: true,
optional_fields: null,
optional_mode: null,
sample_rate: RELAY_CELL_LOG_SAMPLE_RATE
}]
const backendChange = ({ drain = true, logging = true }) => ({
address: 'google_compute_backend_service.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: {
connection_draining_timeout_sec: drain ? 300 : RELAY_CELL_CONNECTION_DRAIN_SECONDS,
log_config: logging ? [] : loggingAfter,
timeout_sec: 86_400,
fingerprint: 'before'
},
after: {
connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS,
log_config: loggingAfter,
timeout_sec: 86_400,
fingerprint: null
},
after_unknown: { fingerprint: true }
}
})
const backend = backendChange({})
const drainOnly = backendChange({ logging: false })
const loggingOnly = backendChange({ drain: false })
const sameCapConfig = {
...config,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: 'relay-director@project.iam.gserviceaccount.com',
rehomeAudience: 'https://relay.onorca.dev/v1/admin/host-drain',
regionalRehomeProtocol: '0'
}
const refused = /only this cell backend drain and request logging/
// Both attributes ride along with the roll without inflating the template-and-MIG count
// the apply's stranded branch and the resume's drift branch both read.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, backend] }, sameCapConfig),
{
mode: 'same-cap-cell',
changes: 2,
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
}
)
// Each is independently optional: a cell that already has one plans no change for it.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, drainOnly] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['connection_draining_timeout_sec'] }
)
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, loggingOnly] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['log_config.0'] }
)
// Once both are applied the backend is simply absent from the plan.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
// A cell whose template and MIG have converged but whose backend has not is still clean.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [backend] }, sameCapConfig),
{
mode: 'same-cap-cell',
changes: 0,
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
}
)
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, sameCapConfig), {
mode: 'same-cap-cell',
changes: 0
})
const extraAttribute = structuredClone(backend)
extraAttribute.change.after.timeout_sec = 3_600
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, extraAttribute] },
sameCapConfig
),
/changes outside the reviewed capacity fields/
)
const otherCell = structuredClone(backend)
otherCell.address = 'google_compute_backend_service.relay_gce_cell["production-gce-c27"]'
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager, otherCell] }, sameCapConfig),
refused
)
const unreviewedDrain = structuredClone(backend)
unreviewedDrain.change.after.connection_draining_timeout_sec =
RELAY_CELL_CONNECTION_DRAIN_SECONDS + 1
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, unreviewedDrain] },
sameCapConfig
),
refused
)
const sampledLogging = structuredClone(backend)
sampledLogging.change.after.log_config[0].sample_rate = RELAY_CELL_LOG_SAMPLE_RATE / 2
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, sampledLogging] },
sameCapConfig
),
refused
)
const disabledLogging = structuredClone(backend)
disabledLogging.change.after.log_config[0].enable = false
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, disabledLogging] },
sameCapConfig
),
refused
)
const replaced = structuredClone(backend)
replaced.change.actions = ['create', 'delete']
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager, replaced] }, sameCapConfig),
refused
)
// Only the same-cap wave targets a backend service; every other mode still refuses one.
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, backend] },
{ ...config, mode: 'cell', image }
),
/only the exact instance template and MIG/
)
})
// The reconciliation allowance is only safe while it names exactly what Terraform declares.
test('the declared MIG fields match relay-gce-cells.tf', () => {
const terraform = readFileSync(
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
'utf8'
)
const manager = terraform
.split('resource "google_compute_instance_group_manager" "relay_gce_cell" {')[1]
?.split('\n}')[0] ?? ''
const block = (name) => manager.split(` ${name} {`)[1]?.split('\n }')[0] ?? ''
const topology = terraform.split(' relay_gce_topology = {')[1]?.split('\n }')[0] ?? ''
const attribute = (source, name) => {
const raw = new RegExp(`\\n\\s+${name}\\s+= (.+)`).exec(source)?.[1]
assert.notEqual(raw, undefined, `relay-gce-cells.tf declares no ${name}`)
const local = /^local\.relay_gce_topology\.(\w+)$/.exec(raw)?.[1]
if (local) return Number(new RegExp(`${local}\\s+= (\\d+)`).exec(topology)?.[1])
return JSON.parse(raw)
}
const declared = Object.fromEntries(
Object.keys(DECLARED_MANAGER_FIELDS).map((path) => {
const [blockName, , field] = path.split('.')
return [path, attribute(block(blockName), field)]
})
)
assert.deepEqual(declared, { ...DECLARED_MANAGER_FIELDS })
})