Files
orca/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
T
Jinwoo Hong acedcf2a97 fix(relay-ops): pin the capacity identity so a stale same-cap template can roll (#21314)
c17's canary-apply failed closed at plan validation. Its instance template is
from 2026-08-07 and predates the ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT line that
every cell rolled since already carries, so the plan legitimately added it. The
same-cap validator holds the whole startup script identical before and after
except the image, and that line is not one it excluded, so the wave stopped
with nothing applied.

Pin the line for same-cap-cell exactly as bootstrap-cell already does, and
exclude it from the before/after comparison. The cell may gain it; the pin is
what refuses a roll that drops it or rewrites it to another identity. Both plan
validations in the job now pass the capacity identity the job already requires.

The same-cap contract is otherwise unchanged: any other stale line still fails
closed, and needs a convergence apply before the cell can roll.
2026-09-17 20:42:34 -04:00

1057 lines
37 KiB
JavaScript

import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
parseCapacityPlanArguments,
validateCapacityPlan as validateCapacityPlanRaw
} from './validate-relay-capacity-plan.mjs'
const config = {
cellId: 'staging-gce-c3',
hardCap: 1_000,
unobservedBound: 60
}
function replacementConfiguration(references = [
'google_compute_instance_template.relay_gce_cell',
'each.key'
]) {
return {
root_module: {
resources: [{
address: 'google_compute_instance_group_manager.relay_gce_cell',
expressions: {
version: [{
instance_template: { references },
name: { constant_value: 'primary' }
}]
}
}]
}
}
}
function validateCapacityPlan(plan, planConfig) {
const replacement = plan.resource_changes.some(({ change }) =>
JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']))
return validateCapacityPlanRaw(
replacement && !plan.configuration
? { ...plan, configuration: replacementConfiguration() }
: plan,
planConfig
)
}
test('accepts only the exact canary template replacement and MIG update', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}`
const startupScript = (cap, bound, selectedImage, extra = '') =>
[
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
extra,
'docker run --detach \\',
' --name cloud-sql-proxy \\',
` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const script = startupScript(1_000, 60, image)
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startupScript(
600,
60,
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
)
},
after: { metadata_startup_script: script, self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const cellConfig = { ...config, mode: 'cell', image }
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), {
mode: 'cell',
changes: 0
})
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
const wrongTemplateManager = structuredClone(manager)
wrongTemplateManager.change.after.version[0].instance_template =
'projects/project/global/instanceTemplates/unreviewed'
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, wrongTemplateManager] },
cellConfig
),
/does not bind the MIG/
)
assert.throws(
() => validateCapacityPlanRaw(
{
resource_changes: [template, manager],
configuration: replacementConfiguration([
'google_compute_instance_template.relay_gce_cell',
'var.unreviewed_key'
])
},
cellConfig
),
/reviewed template dependency/
)
assert.throws(
() => validateCapacityPlanRaw(
{ resource_changes: [template, manager] },
cellConfig
),
/reviewed template dependency/
)
const capacityServiceAccount =
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
const bootstrapTemplate = structuredClone(template)
const bootstrapManager = structuredClone(manager)
bootstrapTemplate.change.after.metadata_startup_script = [
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`,
script
].join('\n')
const bootstrapConfig = {
...cellConfig,
mode: 'bootstrap-cell',
capacityServiceAccount
}
const plannedValues = (startupScript) => ({
root_module: {
resources: [
{
address: bootstrapTemplate.address,
values: {
metadata_startup_script: startupScript,
self_link: 'projects/project/global/instanceTemplates/c26-reviewed'
}
},
{
address: bootstrapManager.address,
values: {
version: [{
instance_template:
'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed'
}]
}
}
]
}
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
const managerOnly = structuredClone(bootstrapManager)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [managerOnly],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 1 }
)
const decoyPlannedValues = plannedValues(
bootstrapTemplate.change.after.metadata_startup_script.replaceAll(
image,
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
) + `\n# decoy '${image}'`
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly], planned_values: decoyPlannedValues },
bootstrapConfig
),
/reviewed image and capacity/
)
const obsoleteTemplate = structuredClone(bootstrapTemplate)
obsoleteTemplate.deposed = 'retired-template'
obsoleteTemplate.change.actions = ['delete']
obsoleteTemplate.change.after = null
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [managerOnly, obsoleteTemplate],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [obsoleteTemplate],
planned_values: plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
},
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 1 }
)
const wrongManagerReference = plannedValues(
bootstrapTemplate.change.after.metadata_startup_script
)
wrongManagerReference.root_module.resources[1].values.version[0].instance_template =
'projects/project/global/instanceTemplates/not-reviewed'
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly], planned_values: wrongManagerReference },
bootstrapConfig
),
/does not bind the MIG/
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [managerOnly] },
bootstrapConfig
),
/no exact planned C26 state/
)
const restartedBootstrapManager = structuredClone(bootstrapManager)
restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }]
restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }]
restartedBootstrapManager.change.before.version[0].name =
'0/2026-08-10 23:30:14.196895+00:00'
restartedBootstrapManager.change.after.version[0].name = 'primary'
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, restartedBootstrapManager] },
bootstrapConfig
),
{ mode: 'bootstrap-cell', changes: 2 }
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [template, restartedBootstrapManager] },
cellConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedRestartManager = structuredClone(restartedBootstrapManager)
unrecognizedRestartManager.change.before.version[0].name = 'operator-version'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedRestartManager] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager)
unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager)
unrecognizedPrimaryVersion.change.after.version[0].name = 'other'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager)
unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const missingRestartPolicy = structuredClone(restartedBootstrapManager)
delete missingRestartPolicy.change.before.update_policy
delete missingRestartPolicy.change.after.update_policy
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, missingRestartPolicy] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const missingRestartVersion = structuredClone(restartedBootstrapManager)
missingRestartVersion.change.before.version[0].name = 'primary'
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, missingRestartVersion] },
bootstrapConfig
),
/outside the reviewed capacity fields/
)
const duplicateHardCapTemplate = structuredClone(template)
duplicateHardCapTemplate.change.after.metadata_startup_script = [
script,
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`
].join('\n')
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] },
cellConfig
),
/reviewed image and capacity/
)
const duplicateIdentityTemplate = structuredClone(bootstrapTemplate)
duplicateIdentityTemplate.change.after.metadata_startup_script = [
duplicateIdentityTemplate.change.after.metadata_startup_script,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'`
].join('\n')
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] },
bootstrapConfig
),
/reviewed image and capacity/
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
{ ...bootstrapConfig, capacityServiceAccount: 'invalid' }
),
/invalid service account/
)
assert.throws(
() =>
validateCapacityPlan(
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
cellConfig
),
/reviewed image and capacity/
)
manager.change.after.target_size = 0
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
manager.change.after.target_size = 1
template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad')
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/reviewed image and capacity/
)
template.change.after.metadata_startup_script = startupScript(
1_000,
60,
image,
'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP='
)
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/reviewed image and capacity/
)
template.change.after.metadata_startup_script = script
template.change.after_unknown = {
id: true,
self_link: true,
disk: [{ architecture: true }]
}
manager.change.after_unknown = {
fingerprint: true,
version: [{ instance_template: true }]
}
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
template.change.before.description = ''
template.change.after.description = null
template.change.before.disk = [{
architecture: '',
source_image: 'projects/cos-cloud/global/images/cos-stable-1'
}]
template.change.after.disk = [{
source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
}]
template.change.after_unknown.disk = [{ architecture: true }]
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
mode: 'cell',
changes: 2
})
template.change.after.disk[0].source_image =
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different'
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
template.change.after.disk[0].source_image =
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
manager.change.after_unknown = { target_size: true }
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
/outside the reviewed capacity fields/
)
})
test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
after: {
metadata_startup_script: startup({ selectedImage: image, trust: true }),
self_link: null
},
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const sameCapConfig = {
...config,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
// A pre-template-apply rollback resume validates drift for the image the
// cell already serves: the template leaves and re-enters the rollback image.
const resumeTemplate = structuredClone(template)
resumeTemplate.change.after.metadata_startup_script = startup({
selectedImage: rollbackImage,
trust: true
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [resumeTemplate, manager] },
{ ...sameCapConfig, image: rollbackImage }
),
{ mode: 'same-cap-cell', changes: 2 }
)
const asiaTemplate = structuredClone(template)
const asiaManager = structuredClone(manager)
asiaTemplate.address =
'google_compute_instance_template.relay_gce_cell["production-gce-c28"]'
asiaManager.address =
'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]'
asiaTemplate.change.before.metadata_startup_script = startup({
selectedImage: rollbackImage,
cap: 3_000
})
asiaTemplate.change.after.metadata_startup_script = startup({
selectedImage: image,
cap: 3_000,
trust: true
})
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [asiaTemplate, asiaManager] },
{ ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 }
),
{ mode: 'same-cap-cell', changes: 2 }
)
const changedCap = structuredClone(template)
changedCap.change.before.metadata_startup_script = startup({
selectedImage: rollbackImage,
cap: 600
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig),
/reviewed image and capacity/
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...sameCapConfig, rollbackImage: image }
),
/reviewed image and capacity/
)
const wrongTrust = structuredClone(template)
wrongTrust.change.after.metadata_startup_script = startup({
selectedImage: image,
trust: true
}).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com')
assert.throws(
() => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig),
/reviewed image and capacity/
)
const imageOnly = structuredClone(template)
imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image })
const imageOnlyConfig = {
...config,
mode: 'same-cap-image',
image,
rollbackImage
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig),
{ mode: 'same-cap-image', changes: 2, changeKind: 'replacement' }
)
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [imageOnly, manager] },
{ ...imageOnlyConfig, rollbackImage: image }
),
/reviewed image and capacity/
)
const changedTrust = structuredClone(imageOnly)
changedTrust.change.after.metadata_startup_script +=
`\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
assert.throws(
() => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig),
/reviewed image and capacity/
)
const plannedValues = {
root_module: {
resources: [
{
address: imageOnly.address,
values: {
metadata_startup_script: imageOnly.change.after.metadata_startup_script,
self_link: 'projects/project/global/instanceTemplates/target'
}
},
{
address: manager.address,
values: {
version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }]
}
}
]
}
}
const obsoleteTemplate = structuredClone(imageOnly)
obsoleteTemplate.deposed = 'obsolete'
obsoleteTemplate.change.actions = ['delete']
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [obsoleteTemplate], planned_values: plannedValues },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' }
)
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate] },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' }
)
const anotherObsoleteTemplate = {
...structuredClone(obsoleteTemplate),
deposed: 'another-obsolete'
}
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [obsoleteTemplate, anotherObsoleteTemplate],
planned_values: plannedValues
},
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' }
)
assert.deepEqual(
validateCapacityPlan(
{
resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate],
planned_values: plannedValues
},
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' }
)
const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate)
invalidObsoleteTemplate.change.actions = ['update']
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] },
imageOnlyConfig
),
/change only the exact instance template and MIG/
)
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [manager], planned_values: plannedValues },
imageOnlyConfig
),
{ mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' }
)
})
test('protocol-0 same-cap cells roll without rehome trust lines', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, trust = false }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
const gainsTrust = structuredClone(template)
gainsTrust.change.after.metadata_startup_script = startup({
selectedImage: image,
trust: true
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig),
/reviewed image and capacity/
)
// Under protocol 1 that same script is the reviewed roll: trust is added, not drift.
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [gainsTrust, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
{ mode: 'same-cap-cell', changes: 2 }
)
// A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged.
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
/reviewed image and capacity/
)
for (const protocol of [undefined, '', '2', 'yes']) {
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: protocol }
),
/invalid regional rehome protocol/
)
}
})
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, capacity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const plan = (beforeCapacity, afterCapacity) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({
selectedImage: rollbackImage,
capacity: beforeCapacity
})
},
after: {
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const config = {
cellId: 'production-gce-c17',
hardCap: 600,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
// A template old enough to predate the line gains it, which is the only move allowed.
assert.deepEqual(
validateCapacityPlan(plan(null, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
for (const [before, after] of [
[capacityIdentity, null],
[null, null],
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
]) {
assert.throws(
() => validateCapacityPlan(plan(before, after), config),
/reviewed image and capacity/,
`${before} -> ${after}`
)
}
// Without the pin there is nothing reviewing the line the comparison now ignores.
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: undefined
}),
/invalid service account/
)
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: 'not-an-email'
}),
/invalid service account/
)
})
test('the capacity identity argument is required by same-cap-cell mode', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const base = [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c17',
'--hard-cap', '600',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '0'
]
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
assert.throws(
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
/--capacity-service-account is invalid/
)
assert.equal(
parseCapacityPlanArguments([
...base,
'--capacity-service-account',
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
]).capacityServiceAccount,
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
)
})
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0'))
.regionalRehomeProtocol,
'0'
)
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
'3'
)
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments()),
/requires rollback image and rehome trust config/
)
for (const protocol of ['', '2', 'true']) {
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)),
/requires rollback image and rehome trust config/
)
}
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--regional-rehome-protocol', '0'
]),
/applies only to same-cap-cell validation/
)
})
test('the reviewed database pool is pinned for the cells that emit one', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, pool }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const rollPlan = (before, after) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
},
after: {
metadata_startup_script: startup({ selectedImage: image, pool: after }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
assert.deepEqual(
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
/reviewed image and capacity/
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
// A cell on the root pool default emits no line at all, and gaining one is real drift.
assert.deepEqual(
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
/reviewed image and capacity/
)
// A line already on the live template is still unreviewed without the pin, even standing still.
assert.throws(
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
/reviewed image and capacity/
)
// A pin must also fail closed when the plan drops the line it names.
assert.throws(
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
for (const pool of ['', '0', '101', 'ten']) {
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
/invalid database pool max/
)
}
})
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '1',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
'16'
)
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--database-pool-max', '16'
]),
/applies only to same-cap-cell validation/
)
})