mirror of
https://github.com/stablyai/orca.git
synced 2026-09-23 16:02:24 +00:00
A same-cap wave applies with create_before_destroy. When run 35684694704 died after the new template was made, the previous template stayed in Terraform state as a deposed object, so every later plan for that cell carried its delete. The plan validator only tolerated deposed deletes in same-cap-image mode, so the recovery run 35698133226 was refused with "cell plan must change only the exact instance template and MIG" and the cell was stranded. Allow exactly one deposed delete at the cell's own template address in same-cap-cell mode too, mirroring the one-deposed bound the convergence path already applies to non-image modes. Report it as `obsoleteTemplates` rather than in `changes`, the way the cell backend update is already split out, so the job's `changes == 2` resume gate and `changes == 0` stranded roll keep reading the template-and-MIG count. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
1303 lines
46 KiB
JavaScript
1303 lines
46 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { test } from 'node:test'
|
|
import {
|
|
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
|
|
RELAY_CELL_LOG_SAMPLE_RATE
|
|
} from './validate-relay-asia-topology-plan.mjs'
|
|
import {
|
|
parseCapacityPlanArguments,
|
|
validateCapacityPlan as validateCapacityPlanRaw
|
|
} from './validate-relay-capacity-plan.mjs'
|
|
|
|
const config = {
|
|
cellId: 'staging-gce-c3',
|
|
hardCap: 1_000,
|
|
unobservedBound: 60
|
|
}
|
|
|
|
function replacementConfiguration(references = [
|
|
'google_compute_instance_template.relay_gce_cell',
|
|
'each.key'
|
|
]) {
|
|
return {
|
|
root_module: {
|
|
resources: [{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell',
|
|
expressions: {
|
|
version: [{
|
|
instance_template: { references },
|
|
name: { constant_value: 'primary' }
|
|
}]
|
|
}
|
|
}]
|
|
}
|
|
}
|
|
}
|
|
|
|
function validateCapacityPlan(plan, planConfig) {
|
|
const replacement = plan.resource_changes.some(({ change }) =>
|
|
JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']))
|
|
return validateCapacityPlanRaw(
|
|
replacement && !plan.configuration
|
|
? { ...plan, configuration: replacementConfiguration() }
|
|
: plan,
|
|
planConfig
|
|
)
|
|
}
|
|
|
|
test('accepts only the exact canary template replacement and MIG update', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}`
|
|
const startupScript = (cap, bound, selectedImage, extra = '') =>
|
|
[
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
extra,
|
|
'docker run --detach \\',
|
|
' --name cloud-sql-proxy \\',
|
|
` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const script = startupScript(1_000, 60, image)
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startupScript(
|
|
600,
|
|
60,
|
|
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
|
|
)
|
|
},
|
|
after: { metadata_startup_script: script, self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const cellConfig = { ...config, mode: 'cell', image }
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 0
|
|
})
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
const wrongTemplateManager = structuredClone(manager)
|
|
wrongTemplateManager.change.after.version[0].instance_template =
|
|
'projects/project/global/instanceTemplates/unreviewed'
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, wrongTemplateManager] },
|
|
cellConfig
|
|
),
|
|
/does not bind the MIG/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlanRaw(
|
|
{
|
|
resource_changes: [template, manager],
|
|
configuration: replacementConfiguration([
|
|
'google_compute_instance_template.relay_gce_cell',
|
|
'var.unreviewed_key'
|
|
])
|
|
},
|
|
cellConfig
|
|
),
|
|
/reviewed template dependency/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlanRaw(
|
|
{ resource_changes: [template, manager] },
|
|
cellConfig
|
|
),
|
|
/reviewed template dependency/
|
|
)
|
|
const capacityServiceAccount =
|
|
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
|
|
const bootstrapTemplate = structuredClone(template)
|
|
const bootstrapManager = structuredClone(manager)
|
|
bootstrapTemplate.change.after.metadata_startup_script = [
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`,
|
|
script
|
|
].join('\n')
|
|
const bootstrapConfig = {
|
|
...cellConfig,
|
|
mode: 'bootstrap-cell',
|
|
capacityServiceAccount
|
|
}
|
|
const plannedValues = (startupScript) => ({
|
|
root_module: {
|
|
resources: [
|
|
{
|
|
address: bootstrapTemplate.address,
|
|
values: {
|
|
metadata_startup_script: startupScript,
|
|
self_link: 'projects/project/global/instanceTemplates/c26-reviewed'
|
|
}
|
|
},
|
|
{
|
|
address: bootstrapManager.address,
|
|
values: {
|
|
version: [{
|
|
instance_template:
|
|
'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed'
|
|
}]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
const managerOnly = structuredClone(bootstrapManager)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [managerOnly],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 1 }
|
|
)
|
|
const decoyPlannedValues = plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script.replaceAll(
|
|
image,
|
|
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
|
|
) + `\n# decoy '${image}'`
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly], planned_values: decoyPlannedValues },
|
|
bootstrapConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const obsoleteTemplate = structuredClone(bootstrapTemplate)
|
|
obsoleteTemplate.deposed = 'retired-template'
|
|
obsoleteTemplate.change.actions = ['delete']
|
|
obsoleteTemplate.change.after = null
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [managerOnly, obsoleteTemplate],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [obsoleteTemplate],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 1 }
|
|
)
|
|
const wrongManagerReference = plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
wrongManagerReference.root_module.resources[1].values.version[0].instance_template =
|
|
'projects/project/global/instanceTemplates/not-reviewed'
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly], planned_values: wrongManagerReference },
|
|
bootstrapConfig
|
|
),
|
|
/does not bind the MIG/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly] },
|
|
bootstrapConfig
|
|
),
|
|
/no exact planned C26 state/
|
|
)
|
|
const restartedBootstrapManager = structuredClone(bootstrapManager)
|
|
restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }]
|
|
restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }]
|
|
restartedBootstrapManager.change.before.version[0].name =
|
|
'0/2026-08-10 23:30:14.196895+00:00'
|
|
restartedBootstrapManager.change.after.version[0].name = 'primary'
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, restartedBootstrapManager] },
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [template, restartedBootstrapManager] },
|
|
cellConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestartManager = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestartManager.change.before.version[0].name = 'operator-version'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestartManager] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager)
|
|
unrecognizedPrimaryVersion.change.after.version[0].name = 'other'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const missingRestartPolicy = structuredClone(restartedBootstrapManager)
|
|
delete missingRestartPolicy.change.before.update_policy
|
|
delete missingRestartPolicy.change.after.update_policy
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, missingRestartPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const missingRestartVersion = structuredClone(restartedBootstrapManager)
|
|
missingRestartVersion.change.before.version[0].name = 'primary'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, missingRestartVersion] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const duplicateHardCapTemplate = structuredClone(template)
|
|
duplicateHardCapTemplate.change.after.metadata_startup_script = [
|
|
script,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`
|
|
].join('\n')
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] },
|
|
cellConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const duplicateIdentityTemplate = structuredClone(bootstrapTemplate)
|
|
duplicateIdentityTemplate.change.after.metadata_startup_script = [
|
|
duplicateIdentityTemplate.change.after.metadata_startup_script,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'`
|
|
].join('\n')
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] },
|
|
bootstrapConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
{ ...bootstrapConfig, capacityServiceAccount: 'invalid' }
|
|
),
|
|
/invalid service account/
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
cellConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
manager.change.after.target_size = 0
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
manager.change.after.target_size = 1
|
|
template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad')
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
template.change.after.metadata_startup_script = startupScript(
|
|
1_000,
|
|
60,
|
|
image,
|
|
'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP='
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
template.change.after.metadata_startup_script = script
|
|
template.change.after_unknown = {
|
|
id: true,
|
|
self_link: true,
|
|
disk: [{ architecture: true }]
|
|
}
|
|
manager.change.after_unknown = {
|
|
fingerprint: true,
|
|
version: [{ instance_template: true }]
|
|
}
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
template.change.before.description = ''
|
|
template.change.after.description = null
|
|
template.change.before.disk = [{
|
|
architecture: '',
|
|
source_image: 'projects/cos-cloud/global/images/cos-stable-1'
|
|
}]
|
|
template.change.after.disk = [{
|
|
source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
|
|
}]
|
|
template.change.after_unknown.disk = [{ architecture: true }]
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
template.change.after.disk[0].source_image =
|
|
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different'
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
template.change.after.disk[0].source_image =
|
|
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
|
|
manager.change.after_unknown = { target_size: true }
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
})
|
|
|
|
test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
...(capacity === null
|
|
? []
|
|
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
|
...(trust ? [
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
] : []),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, trust: true }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const sameCapConfig = {
|
|
...config,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '1'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A pre-template-apply rollback resume validates drift for the image the
|
|
// cell already serves: the template leaves and re-enters the rollback image.
|
|
const resumeTemplate = structuredClone(template)
|
|
resumeTemplate.change.after.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
trust: true
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [resumeTemplate, manager] },
|
|
{ ...sameCapConfig, image: rollbackImage }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const asiaTemplate = structuredClone(template)
|
|
const asiaManager = structuredClone(manager)
|
|
asiaTemplate.address =
|
|
'google_compute_instance_template.relay_gce_cell["production-gce-c28"]'
|
|
asiaManager.address =
|
|
'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]'
|
|
asiaTemplate.change.before.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
cap: 3_000
|
|
})
|
|
asiaTemplate.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
cap: 3_000,
|
|
trust: true
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [asiaTemplate, asiaManager] },
|
|
{ ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const changedCap = structuredClone(template)
|
|
changedCap.change.before.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
cap: 600
|
|
})
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...sameCapConfig, rollbackImage: image }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const wrongTrust = structuredClone(template)
|
|
wrongTrust.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
trust: true
|
|
}).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com')
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
|
|
// Production run 35684694704 died after create_before_destroy made the new template, leaving
|
|
// the Sep 18 template deposed; every later wave plan for that cell carries its delete.
|
|
const deposedTemplate = structuredClone(template)
|
|
deposedTemplate.deposed = '4cb19f83'
|
|
deposedTemplate.change.actions = ['delete']
|
|
deposedTemplate.change.after = null
|
|
deposedTemplate.change.after_unknown = {}
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [template, manager, deposedTemplate] },
|
|
sameCapConfig
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2, obsoleteTemplates: 1 }
|
|
)
|
|
const updatedDeposedTemplate = structuredClone(deposedTemplate)
|
|
updatedDeposedTemplate.change.actions = ['update']
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, updatedDeposedTemplate] },
|
|
sameCapConfig
|
|
),
|
|
/change only the exact instance template and MIG/
|
|
)
|
|
const foreignDeposedTemplate = structuredClone(deposedTemplate)
|
|
foreignDeposedTemplate.address =
|
|
'google_compute_instance_template.relay_gce_cell["production-gce-c8"]'
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, foreignDeposedTemplate] },
|
|
sameCapConfig
|
|
),
|
|
/change only the exact instance template and MIG/
|
|
)
|
|
const secondDeposedTemplate = { ...structuredClone(deposedTemplate), deposed: 'aa17b204' }
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, deposedTemplate, secondDeposedTemplate] },
|
|
sameCapConfig
|
|
),
|
|
/change only the exact instance template and MIG/
|
|
)
|
|
// Nothing but the deposed delete left: the cell is stranded on the reviewed template, and the
|
|
// job's `changes == 0` roll must still fire.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [deposedTemplate],
|
|
planned_values: {
|
|
root_module: {
|
|
resources: [
|
|
{
|
|
address: template.address,
|
|
values: {
|
|
metadata_startup_script: template.change.after.metadata_startup_script,
|
|
self_link: 'projects/project/global/instanceTemplates/target'
|
|
}
|
|
},
|
|
{
|
|
address: manager.address,
|
|
values: {
|
|
version: [{
|
|
instance_template: 'projects/project/global/instanceTemplates/target'
|
|
}]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
sameCapConfig
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 0, obsoleteTemplates: 1 }
|
|
)
|
|
|
|
const imageOnly = structuredClone(template)
|
|
imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image })
|
|
const imageOnlyConfig = {
|
|
...config,
|
|
mode: 'same-cap-image',
|
|
image,
|
|
rollbackImage
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig),
|
|
{ mode: 'same-cap-image', changes: 2, changeKind: 'replacement' }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager] },
|
|
{ ...imageOnlyConfig, rollbackImage: image }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const changedTrust = structuredClone(imageOnly)
|
|
changedTrust.change.after.metadata_startup_script +=
|
|
`\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
const plannedValues = {
|
|
root_module: {
|
|
resources: [
|
|
{
|
|
address: imageOnly.address,
|
|
values: {
|
|
metadata_startup_script: imageOnly.change.after.metadata_startup_script,
|
|
self_link: 'projects/project/global/instanceTemplates/target'
|
|
}
|
|
},
|
|
{
|
|
address: manager.address,
|
|
values: {
|
|
version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
const obsoleteTemplate = structuredClone(imageOnly)
|
|
obsoleteTemplate.deposed = 'obsolete'
|
|
obsoleteTemplate.change.actions = ['delete']
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [obsoleteTemplate], planned_values: plannedValues },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' }
|
|
)
|
|
const anotherObsoleteTemplate = {
|
|
...structuredClone(obsoleteTemplate),
|
|
deposed: 'another-obsolete'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [obsoleteTemplate, anotherObsoleteTemplate],
|
|
planned_values: plannedValues
|
|
},
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate],
|
|
planned_values: plannedValues
|
|
},
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' }
|
|
)
|
|
const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate)
|
|
invalidObsoleteTemplate.change.actions = ['update']
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
/change only the exact instance template and MIG/
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [manager], planned_values: plannedValues },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' }
|
|
)
|
|
})
|
|
|
|
test('protocol-0 same-cap cells roll without rehome trust lines', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, trust = false }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
|
|
...(trust ? [
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
] : []),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
|
|
after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const asiaConfig = {
|
|
cellId: 'production-gce-c27',
|
|
hardCap: 3_000,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const gainsTrust = structuredClone(template)
|
|
gainsTrust.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
trust: true
|
|
})
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// Under protocol 1 that same script is the reviewed roll: trust is added, not drift.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [gainsTrust, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: '1' }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged.
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: '1' }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
for (const protocol of [undefined, '', '2', 'yes']) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: protocol }
|
|
),
|
|
/invalid regional rehome protocol/
|
|
)
|
|
}
|
|
})
|
|
|
|
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, capacity }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
...(capacity === null
|
|
? []
|
|
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const plan = (beforeCapacity, afterCapacity) => ({
|
|
resource_changes: [
|
|
{
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startup({
|
|
selectedImage: rollbackImage,
|
|
capacity: beforeCapacity
|
|
})
|
|
},
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
},
|
|
{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
]
|
|
})
|
|
const config = {
|
|
cellId: 'production-gce-c17',
|
|
hardCap: 600,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
// A template old enough to predate the line gains it, which is the only move allowed.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(plan(null, capacityIdentity), config),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
for (const [before, after] of [
|
|
[capacityIdentity, null],
|
|
[null, null],
|
|
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
|
|
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
|
|
]) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(before, after), config),
|
|
/reviewed image and capacity/,
|
|
`${before} -> ${after}`
|
|
)
|
|
}
|
|
// Without the pin there is nothing reviewing the line the comparison now ignores.
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
|
...config,
|
|
capacityServiceAccount: undefined
|
|
}),
|
|
/invalid service account/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
|
...config,
|
|
capacityServiceAccount: 'not-an-email'
|
|
}),
|
|
/invalid service account/
|
|
)
|
|
})
|
|
|
|
test('the capacity identity argument is required by same-cap-cell mode', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const base = [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c17',
|
|
'--hard-cap', '600',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
'--regional-rehome-protocol', '0'
|
|
]
|
|
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
|
|
/--capacity-service-account is invalid/
|
|
)
|
|
assert.equal(
|
|
parseCapacityPlanArguments([
|
|
...base,
|
|
'--capacity-service-account',
|
|
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
]).capacityServiceAccount,
|
|
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
)
|
|
})
|
|
|
|
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const sameCapArguments = (...extra) => [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c27',
|
|
'--hard-cap', '3000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
...extra
|
|
]
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0'))
|
|
.regionalRehomeProtocol,
|
|
'0'
|
|
)
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
|
|
'3'
|
|
)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments(sameCapArguments()),
|
|
/requires rollback image and rehome trust config/
|
|
)
|
|
for (const protocol of ['', '2', 'true']) {
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)),
|
|
/requires rollback image and rehome trust config/
|
|
)
|
|
}
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([
|
|
'--mode', 'bootstrap-cell',
|
|
'--cell-id', 'staging-gce-c3',
|
|
'--hard-cap', '1000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
|
|
'--regional-rehome-protocol', '0'
|
|
]),
|
|
/applies only to same-cap-cell validation/
|
|
)
|
|
})
|
|
|
|
test('the reviewed database pool is pinned for the cells that emit one', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, pool }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
...(pool === undefined
|
|
? []
|
|
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const rollPlan = (before, after) => ({
|
|
resource_changes: [
|
|
{
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
|
|
},
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, pool: after }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
},
|
|
{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
]
|
|
})
|
|
const asiaConfig = {
|
|
cellId: 'production-gce-c27',
|
|
hardCap: 3_000,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '1'
|
|
}
|
|
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A cell on the root pool default emits no line at all, and gaining one is real drift.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A line already on the live template is still unreviewed without the pin, even standing still.
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A pin must also fail closed when the plan drops the line it names.
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
|
|
/reviewed image and capacity/
|
|
)
|
|
for (const pool of ['', '0', '101', 'ten']) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
|
|
/invalid database pool max/
|
|
)
|
|
}
|
|
})
|
|
|
|
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const sameCapArguments = (...extra) => [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c27',
|
|
'--hard-cap', '3000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
'--regional-rehome-protocol', '1',
|
|
...extra
|
|
]
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
|
|
'16'
|
|
)
|
|
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([
|
|
'--mode', 'bootstrap-cell',
|
|
'--cell-id', 'staging-gce-c3',
|
|
'--hard-cap', '1000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
|
|
'--database-pool-max', '16'
|
|
]),
|
|
/applies only to same-cap-cell validation/
|
|
)
|
|
})
|
|
|
|
test('a same-cap roll may carry only this cell backend drain and request logging', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const startup = (selectedImage) => [
|
|
" printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '1000'",
|
|
" printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'",
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup(rollbackImage) },
|
|
after: { metadata_startup_script: startup(image), self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
// The exact shape a live US cell's backend plans: 300 s drain and no log_config at all.
|
|
const loggingAfter = [{
|
|
enable: true,
|
|
optional_fields: null,
|
|
optional_mode: null,
|
|
sample_rate: RELAY_CELL_LOG_SAMPLE_RATE
|
|
}]
|
|
const backendChange = ({ drain = true, logging = true }) => ({
|
|
address: 'google_compute_backend_service.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: {
|
|
connection_draining_timeout_sec: drain ? 300 : RELAY_CELL_CONNECTION_DRAIN_SECONDS,
|
|
log_config: logging ? [] : loggingAfter,
|
|
timeout_sec: 86_400,
|
|
fingerprint: 'before'
|
|
},
|
|
after: {
|
|
connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS,
|
|
log_config: loggingAfter,
|
|
timeout_sec: 86_400,
|
|
fingerprint: null
|
|
},
|
|
after_unknown: { fingerprint: true }
|
|
}
|
|
})
|
|
const backend = backendChange({})
|
|
const drainOnly = backendChange({ logging: false })
|
|
const loggingOnly = backendChange({ drain: false })
|
|
const sameCapConfig = {
|
|
...config,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: 'relay-director@project.iam.gserviceaccount.com',
|
|
rehomeAudience: 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
const refused = /only this cell backend drain and request logging/
|
|
// Both attributes ride along with the roll without inflating the template-and-MIG count
|
|
// the apply's stranded branch and the resume's drift branch both read.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager, backend] }, sameCapConfig),
|
|
{
|
|
mode: 'same-cap-cell',
|
|
changes: 2,
|
|
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
|
|
}
|
|
)
|
|
// Each is independently optional: a cell that already has one plans no change for it.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager, drainOnly] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['connection_draining_timeout_sec'] }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager, loggingOnly] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['log_config.0'] }
|
|
)
|
|
// Once both are applied the backend is simply absent from the plan.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A cell whose template and MIG have converged but whose backend has not is still clean.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [backend] }, sameCapConfig),
|
|
{
|
|
mode: 'same-cap-cell',
|
|
changes: 0,
|
|
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
|
|
}
|
|
)
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, sameCapConfig), {
|
|
mode: 'same-cap-cell',
|
|
changes: 0
|
|
})
|
|
const extraAttribute = structuredClone(backend)
|
|
extraAttribute.change.after.timeout_sec = 3_600
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, extraAttribute] },
|
|
sameCapConfig
|
|
),
|
|
/changes outside the reviewed capacity fields/
|
|
)
|
|
const otherCell = structuredClone(backend)
|
|
otherCell.address = 'google_compute_backend_service.relay_gce_cell["production-gce-c27"]'
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager, otherCell] }, sameCapConfig),
|
|
refused
|
|
)
|
|
const unreviewedDrain = structuredClone(backend)
|
|
unreviewedDrain.change.after.connection_draining_timeout_sec =
|
|
RELAY_CELL_CONNECTION_DRAIN_SECONDS + 1
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, unreviewedDrain] },
|
|
sameCapConfig
|
|
),
|
|
refused
|
|
)
|
|
const sampledLogging = structuredClone(backend)
|
|
sampledLogging.change.after.log_config[0].sample_rate = RELAY_CELL_LOG_SAMPLE_RATE / 2
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, sampledLogging] },
|
|
sameCapConfig
|
|
),
|
|
refused
|
|
)
|
|
const disabledLogging = structuredClone(backend)
|
|
disabledLogging.change.after.log_config[0].enable = false
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, disabledLogging] },
|
|
sameCapConfig
|
|
),
|
|
refused
|
|
)
|
|
const replaced = structuredClone(backend)
|
|
replaced.change.actions = ['create', 'delete']
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager, replaced] }, sameCapConfig),
|
|
refused
|
|
)
|
|
// Only the same-cap wave targets a backend service; every other mode still refuses one.
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, backend] },
|
|
{ ...config, mode: 'cell', image }
|
|
),
|
|
/only the exact instance template and MIG/
|
|
)
|
|
})
|