mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 16:02:32 +00:00
* chore(relay): target each cell's backend service from the same-cap job
The 60 s connection drain timeout merged in #21848 has no safe apply path.
A root plan scoped to the backend services alone still pulls every
`google_compute_instance_template.relay_gce_cell` in as a dependency, and
standing image drift turns all 29 into replacements, so applying it would roll
the fleet at once.
Add `google_compute_backend_service.relay_gce_cell["${TARGET_CELL_ID}"]` to
both plan invocations in the per-cell same-cap job, next to the template and
MIG it already targets, and teach the reviewed plan validator to allow exactly
one extra change: an in-place update of that one cell's backend whose only
changed attribute is `connection_draining_timeout_sec`, landing on the
constant `validate-relay-asia-topology-plan.mjs` exports. Any other attribute,
any other resource, or a backend for another cell still fails the validator.
The accepted update is reported as `connectionDrainUpdate` and kept out of
`changes`, so the apply step's stranded branch and the resume step's drift
branch keep reading the template-and-MIG count they were written against; the
resume branch additionally accepts a plan whose only pending change is that
drain update, which restarts nothing.
Claude-Session: relay-same-cap-targets-cell-backend
* fix(relay): also let the same-cap wave apply this cell's LB request logging
A read-only production plan for production-gce-c7 showed the live US cell
backends carry no `log_config` at all, while relay-gce-cells.tf has declared
`log_config { enable = true, sample_rate = var.relay_gce_cell_log_sample_rate }`
on every cell backend since the Terraform root landed in 3eec77c11a (#18413).
Nothing has applied it because every production apply since is a per-cell
targeted plan that names only the template and the MIG.
So the real canary plan's backend moves two paths, not one:
`["connection_draining_timeout_sec", "log_config.0"]`. The drain-only validator
rejected exactly that plan, which would have stranded the cell mid-wave after
the drain had already started.
Accept both, each optional, for this cell's backend only: the drain landing on
RELAY_CELL_CONNECTION_DRAIN_SECONDS, and a log_config of exactly one block with
`enable = true` and `sample_rate` equal to RELAY_CELL_LOG_SAMPLE_RATE, the
declared default of a variable no environment file overrides. Any third path,
a different sample rate, disabled logging, another cell, or a replacement still
fails. The accepted paths are reported as `backendUpdate`, which the resume
branch now reads instead of the drain-only flag.
Verified against the real production plan: the masked, three-target plan for
production-gce-c7 contains exactly that cell's template, MIG, and backend
service and nothing else, and this validator returns
`{"changes":2,"backendUpdate":["connection_draining_timeout_sec","log_config.0"]}`.
Claude-Session: relay-same-cap-targets-cell-backend
1229 lines
44 KiB
JavaScript
1229 lines
44 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { test } from 'node:test'
|
|
import {
|
|
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
|
|
RELAY_CELL_LOG_SAMPLE_RATE
|
|
} from './validate-relay-asia-topology-plan.mjs'
|
|
import {
|
|
parseCapacityPlanArguments,
|
|
validateCapacityPlan as validateCapacityPlanRaw
|
|
} from './validate-relay-capacity-plan.mjs'
|
|
|
|
const config = {
|
|
cellId: 'staging-gce-c3',
|
|
hardCap: 1_000,
|
|
unobservedBound: 60
|
|
}
|
|
|
|
function replacementConfiguration(references = [
|
|
'google_compute_instance_template.relay_gce_cell',
|
|
'each.key'
|
|
]) {
|
|
return {
|
|
root_module: {
|
|
resources: [{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell',
|
|
expressions: {
|
|
version: [{
|
|
instance_template: { references },
|
|
name: { constant_value: 'primary' }
|
|
}]
|
|
}
|
|
}]
|
|
}
|
|
}
|
|
}
|
|
|
|
function validateCapacityPlan(plan, planConfig) {
|
|
const replacement = plan.resource_changes.some(({ change }) =>
|
|
JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']))
|
|
return validateCapacityPlanRaw(
|
|
replacement && !plan.configuration
|
|
? { ...plan, configuration: replacementConfiguration() }
|
|
: plan,
|
|
planConfig
|
|
)
|
|
}
|
|
|
|
test('accepts only the exact canary template replacement and MIG update', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}`
|
|
const startupScript = (cap, bound, selectedImage, extra = '') =>
|
|
[
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
extra,
|
|
'docker run --detach \\',
|
|
' --name cloud-sql-proxy \\',
|
|
` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const script = startupScript(1_000, 60, image)
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startupScript(
|
|
600,
|
|
60,
|
|
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
|
|
)
|
|
},
|
|
after: { metadata_startup_script: script, self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const cellConfig = { ...config, mode: 'cell', image }
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 0
|
|
})
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
const wrongTemplateManager = structuredClone(manager)
|
|
wrongTemplateManager.change.after.version[0].instance_template =
|
|
'projects/project/global/instanceTemplates/unreviewed'
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, wrongTemplateManager] },
|
|
cellConfig
|
|
),
|
|
/does not bind the MIG/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlanRaw(
|
|
{
|
|
resource_changes: [template, manager],
|
|
configuration: replacementConfiguration([
|
|
'google_compute_instance_template.relay_gce_cell',
|
|
'var.unreviewed_key'
|
|
])
|
|
},
|
|
cellConfig
|
|
),
|
|
/reviewed template dependency/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlanRaw(
|
|
{ resource_changes: [template, manager] },
|
|
cellConfig
|
|
),
|
|
/reviewed template dependency/
|
|
)
|
|
const capacityServiceAccount =
|
|
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
|
|
const bootstrapTemplate = structuredClone(template)
|
|
const bootstrapManager = structuredClone(manager)
|
|
bootstrapTemplate.change.after.metadata_startup_script = [
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`,
|
|
script
|
|
].join('\n')
|
|
const bootstrapConfig = {
|
|
...cellConfig,
|
|
mode: 'bootstrap-cell',
|
|
capacityServiceAccount
|
|
}
|
|
const plannedValues = (startupScript) => ({
|
|
root_module: {
|
|
resources: [
|
|
{
|
|
address: bootstrapTemplate.address,
|
|
values: {
|
|
metadata_startup_script: startupScript,
|
|
self_link: 'projects/project/global/instanceTemplates/c26-reviewed'
|
|
}
|
|
},
|
|
{
|
|
address: bootstrapManager.address,
|
|
values: {
|
|
version: [{
|
|
instance_template:
|
|
'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed'
|
|
}]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
const managerOnly = structuredClone(bootstrapManager)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [managerOnly],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 1 }
|
|
)
|
|
const decoyPlannedValues = plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script.replaceAll(
|
|
image,
|
|
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
|
|
) + `\n# decoy '${image}'`
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly], planned_values: decoyPlannedValues },
|
|
bootstrapConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const obsoleteTemplate = structuredClone(bootstrapTemplate)
|
|
obsoleteTemplate.deposed = 'retired-template'
|
|
obsoleteTemplate.change.actions = ['delete']
|
|
obsoleteTemplate.change.after = null
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [managerOnly, obsoleteTemplate],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [obsoleteTemplate],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 1 }
|
|
)
|
|
const wrongManagerReference = plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
wrongManagerReference.root_module.resources[1].values.version[0].instance_template =
|
|
'projects/project/global/instanceTemplates/not-reviewed'
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly], planned_values: wrongManagerReference },
|
|
bootstrapConfig
|
|
),
|
|
/does not bind the MIG/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly] },
|
|
bootstrapConfig
|
|
),
|
|
/no exact planned C26 state/
|
|
)
|
|
const restartedBootstrapManager = structuredClone(bootstrapManager)
|
|
restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }]
|
|
restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }]
|
|
restartedBootstrapManager.change.before.version[0].name =
|
|
'0/2026-08-10 23:30:14.196895+00:00'
|
|
restartedBootstrapManager.change.after.version[0].name = 'primary'
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, restartedBootstrapManager] },
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [template, restartedBootstrapManager] },
|
|
cellConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestartManager = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestartManager.change.before.version[0].name = 'operator-version'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestartManager] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager)
|
|
unrecognizedPrimaryVersion.change.after.version[0].name = 'other'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const missingRestartPolicy = structuredClone(restartedBootstrapManager)
|
|
delete missingRestartPolicy.change.before.update_policy
|
|
delete missingRestartPolicy.change.after.update_policy
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, missingRestartPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const missingRestartVersion = structuredClone(restartedBootstrapManager)
|
|
missingRestartVersion.change.before.version[0].name = 'primary'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, missingRestartVersion] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const duplicateHardCapTemplate = structuredClone(template)
|
|
duplicateHardCapTemplate.change.after.metadata_startup_script = [
|
|
script,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`
|
|
].join('\n')
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] },
|
|
cellConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const duplicateIdentityTemplate = structuredClone(bootstrapTemplate)
|
|
duplicateIdentityTemplate.change.after.metadata_startup_script = [
|
|
duplicateIdentityTemplate.change.after.metadata_startup_script,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'`
|
|
].join('\n')
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] },
|
|
bootstrapConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
{ ...bootstrapConfig, capacityServiceAccount: 'invalid' }
|
|
),
|
|
/invalid service account/
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
cellConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
manager.change.after.target_size = 0
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
manager.change.after.target_size = 1
|
|
template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad')
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
template.change.after.metadata_startup_script = startupScript(
|
|
1_000,
|
|
60,
|
|
image,
|
|
'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP='
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
template.change.after.metadata_startup_script = script
|
|
template.change.after_unknown = {
|
|
id: true,
|
|
self_link: true,
|
|
disk: [{ architecture: true }]
|
|
}
|
|
manager.change.after_unknown = {
|
|
fingerprint: true,
|
|
version: [{ instance_template: true }]
|
|
}
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
template.change.before.description = ''
|
|
template.change.after.description = null
|
|
template.change.before.disk = [{
|
|
architecture: '',
|
|
source_image: 'projects/cos-cloud/global/images/cos-stable-1'
|
|
}]
|
|
template.change.after.disk = [{
|
|
source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
|
|
}]
|
|
template.change.after_unknown.disk = [{ architecture: true }]
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
template.change.after.disk[0].source_image =
|
|
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different'
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
template.change.after.disk[0].source_image =
|
|
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
|
|
manager.change.after_unknown = { target_size: true }
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
})
|
|
|
|
test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
...(capacity === null
|
|
? []
|
|
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
|
...(trust ? [
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
] : []),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, trust: true }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const sameCapConfig = {
|
|
...config,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '1'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A pre-template-apply rollback resume validates drift for the image the
|
|
// cell already serves: the template leaves and re-enters the rollback image.
|
|
const resumeTemplate = structuredClone(template)
|
|
resumeTemplate.change.after.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
trust: true
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [resumeTemplate, manager] },
|
|
{ ...sameCapConfig, image: rollbackImage }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const asiaTemplate = structuredClone(template)
|
|
const asiaManager = structuredClone(manager)
|
|
asiaTemplate.address =
|
|
'google_compute_instance_template.relay_gce_cell["production-gce-c28"]'
|
|
asiaManager.address =
|
|
'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]'
|
|
asiaTemplate.change.before.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
cap: 3_000
|
|
})
|
|
asiaTemplate.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
cap: 3_000,
|
|
trust: true
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [asiaTemplate, asiaManager] },
|
|
{ ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const changedCap = structuredClone(template)
|
|
changedCap.change.before.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
cap: 600
|
|
})
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...sameCapConfig, rollbackImage: image }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const wrongTrust = structuredClone(template)
|
|
wrongTrust.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
trust: true
|
|
}).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com')
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
|
|
const imageOnly = structuredClone(template)
|
|
imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image })
|
|
const imageOnlyConfig = {
|
|
...config,
|
|
mode: 'same-cap-image',
|
|
image,
|
|
rollbackImage
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig),
|
|
{ mode: 'same-cap-image', changes: 2, changeKind: 'replacement' }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager] },
|
|
{ ...imageOnlyConfig, rollbackImage: image }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const changedTrust = structuredClone(imageOnly)
|
|
changedTrust.change.after.metadata_startup_script +=
|
|
`\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
const plannedValues = {
|
|
root_module: {
|
|
resources: [
|
|
{
|
|
address: imageOnly.address,
|
|
values: {
|
|
metadata_startup_script: imageOnly.change.after.metadata_startup_script,
|
|
self_link: 'projects/project/global/instanceTemplates/target'
|
|
}
|
|
},
|
|
{
|
|
address: manager.address,
|
|
values: {
|
|
version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
const obsoleteTemplate = structuredClone(imageOnly)
|
|
obsoleteTemplate.deposed = 'obsolete'
|
|
obsoleteTemplate.change.actions = ['delete']
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [obsoleteTemplate], planned_values: plannedValues },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' }
|
|
)
|
|
const anotherObsoleteTemplate = {
|
|
...structuredClone(obsoleteTemplate),
|
|
deposed: 'another-obsolete'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [obsoleteTemplate, anotherObsoleteTemplate],
|
|
planned_values: plannedValues
|
|
},
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate],
|
|
planned_values: plannedValues
|
|
},
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' }
|
|
)
|
|
const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate)
|
|
invalidObsoleteTemplate.change.actions = ['update']
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
/change only the exact instance template and MIG/
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [manager], planned_values: plannedValues },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' }
|
|
)
|
|
})
|
|
|
|
test('protocol-0 same-cap cells roll without rehome trust lines', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, trust = false }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
|
|
...(trust ? [
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
] : []),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
|
|
after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const asiaConfig = {
|
|
cellId: 'production-gce-c27',
|
|
hardCap: 3_000,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const gainsTrust = structuredClone(template)
|
|
gainsTrust.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
trust: true
|
|
})
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// Under protocol 1 that same script is the reviewed roll: trust is added, not drift.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [gainsTrust, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: '1' }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged.
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: '1' }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
for (const protocol of [undefined, '', '2', 'yes']) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: protocol }
|
|
),
|
|
/invalid regional rehome protocol/
|
|
)
|
|
}
|
|
})
|
|
|
|
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, capacity }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
...(capacity === null
|
|
? []
|
|
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const plan = (beforeCapacity, afterCapacity) => ({
|
|
resource_changes: [
|
|
{
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startup({
|
|
selectedImage: rollbackImage,
|
|
capacity: beforeCapacity
|
|
})
|
|
},
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
},
|
|
{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
]
|
|
})
|
|
const config = {
|
|
cellId: 'production-gce-c17',
|
|
hardCap: 600,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
// A template old enough to predate the line gains it, which is the only move allowed.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(plan(null, capacityIdentity), config),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
for (const [before, after] of [
|
|
[capacityIdentity, null],
|
|
[null, null],
|
|
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
|
|
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
|
|
]) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(before, after), config),
|
|
/reviewed image and capacity/,
|
|
`${before} -> ${after}`
|
|
)
|
|
}
|
|
// Without the pin there is nothing reviewing the line the comparison now ignores.
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
|
...config,
|
|
capacityServiceAccount: undefined
|
|
}),
|
|
/invalid service account/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
|
...config,
|
|
capacityServiceAccount: 'not-an-email'
|
|
}),
|
|
/invalid service account/
|
|
)
|
|
})
|
|
|
|
test('the capacity identity argument is required by same-cap-cell mode', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const base = [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c17',
|
|
'--hard-cap', '600',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
'--regional-rehome-protocol', '0'
|
|
]
|
|
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
|
|
/--capacity-service-account is invalid/
|
|
)
|
|
assert.equal(
|
|
parseCapacityPlanArguments([
|
|
...base,
|
|
'--capacity-service-account',
|
|
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
]).capacityServiceAccount,
|
|
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
)
|
|
})
|
|
|
|
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const sameCapArguments = (...extra) => [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c27',
|
|
'--hard-cap', '3000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
...extra
|
|
]
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0'))
|
|
.regionalRehomeProtocol,
|
|
'0'
|
|
)
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
|
|
'3'
|
|
)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments(sameCapArguments()),
|
|
/requires rollback image and rehome trust config/
|
|
)
|
|
for (const protocol of ['', '2', 'true']) {
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)),
|
|
/requires rollback image and rehome trust config/
|
|
)
|
|
}
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([
|
|
'--mode', 'bootstrap-cell',
|
|
'--cell-id', 'staging-gce-c3',
|
|
'--hard-cap', '1000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
|
|
'--regional-rehome-protocol', '0'
|
|
]),
|
|
/applies only to same-cap-cell validation/
|
|
)
|
|
})
|
|
|
|
test('the reviewed database pool is pinned for the cells that emit one', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, pool }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
...(pool === undefined
|
|
? []
|
|
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const rollPlan = (before, after) => ({
|
|
resource_changes: [
|
|
{
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
|
|
},
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, pool: after }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
},
|
|
{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
]
|
|
})
|
|
const asiaConfig = {
|
|
cellId: 'production-gce-c27',
|
|
hardCap: 3_000,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '1'
|
|
}
|
|
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A cell on the root pool default emits no line at all, and gaining one is real drift.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A line already on the live template is still unreviewed without the pin, even standing still.
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A pin must also fail closed when the plan drops the line it names.
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
|
|
/reviewed image and capacity/
|
|
)
|
|
for (const pool of ['', '0', '101', 'ten']) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
|
|
/invalid database pool max/
|
|
)
|
|
}
|
|
})
|
|
|
|
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const sameCapArguments = (...extra) => [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c27',
|
|
'--hard-cap', '3000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
'--regional-rehome-protocol', '1',
|
|
...extra
|
|
]
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
|
|
'16'
|
|
)
|
|
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([
|
|
'--mode', 'bootstrap-cell',
|
|
'--cell-id', 'staging-gce-c3',
|
|
'--hard-cap', '1000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
|
|
'--database-pool-max', '16'
|
|
]),
|
|
/applies only to same-cap-cell validation/
|
|
)
|
|
})
|
|
|
|
test('a same-cap roll may carry only this cell backend drain and request logging', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const startup = (selectedImage) => [
|
|
" printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '1000'",
|
|
" printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'",
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup(rollbackImage) },
|
|
after: { metadata_startup_script: startup(image), self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
// The exact shape a live US cell's backend plans: 300 s drain and no log_config at all.
|
|
const loggingAfter = [{
|
|
enable: true,
|
|
optional_fields: null,
|
|
optional_mode: null,
|
|
sample_rate: RELAY_CELL_LOG_SAMPLE_RATE
|
|
}]
|
|
const backendChange = ({ drain = true, logging = true }) => ({
|
|
address: 'google_compute_backend_service.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: {
|
|
connection_draining_timeout_sec: drain ? 300 : RELAY_CELL_CONNECTION_DRAIN_SECONDS,
|
|
log_config: logging ? [] : loggingAfter,
|
|
timeout_sec: 86_400,
|
|
fingerprint: 'before'
|
|
},
|
|
after: {
|
|
connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS,
|
|
log_config: loggingAfter,
|
|
timeout_sec: 86_400,
|
|
fingerprint: null
|
|
},
|
|
after_unknown: { fingerprint: true }
|
|
}
|
|
})
|
|
const backend = backendChange({})
|
|
const drainOnly = backendChange({ logging: false })
|
|
const loggingOnly = backendChange({ drain: false })
|
|
const sameCapConfig = {
|
|
...config,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: 'relay-director@project.iam.gserviceaccount.com',
|
|
rehomeAudience: 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
const refused = /only this cell backend drain and request logging/
|
|
// Both attributes ride along with the roll without inflating the template-and-MIG count
|
|
// the apply's stranded branch and the resume's drift branch both read.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager, backend] }, sameCapConfig),
|
|
{
|
|
mode: 'same-cap-cell',
|
|
changes: 2,
|
|
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
|
|
}
|
|
)
|
|
// Each is independently optional: a cell that already has one plans no change for it.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager, drainOnly] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['connection_draining_timeout_sec'] }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager, loggingOnly] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['log_config.0'] }
|
|
)
|
|
// Once both are applied the backend is simply absent from the plan.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A cell whose template and MIG have converged but whose backend has not is still clean.
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [backend] }, sameCapConfig),
|
|
{
|
|
mode: 'same-cap-cell',
|
|
changes: 0,
|
|
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
|
|
}
|
|
)
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, sameCapConfig), {
|
|
mode: 'same-cap-cell',
|
|
changes: 0
|
|
})
|
|
const extraAttribute = structuredClone(backend)
|
|
extraAttribute.change.after.timeout_sec = 3_600
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, extraAttribute] },
|
|
sameCapConfig
|
|
),
|
|
/changes outside the reviewed capacity fields/
|
|
)
|
|
const otherCell = structuredClone(backend)
|
|
otherCell.address = 'google_compute_backend_service.relay_gce_cell["production-gce-c27"]'
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager, otherCell] }, sameCapConfig),
|
|
refused
|
|
)
|
|
const unreviewedDrain = structuredClone(backend)
|
|
unreviewedDrain.change.after.connection_draining_timeout_sec =
|
|
RELAY_CELL_CONNECTION_DRAIN_SECONDS + 1
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, unreviewedDrain] },
|
|
sameCapConfig
|
|
),
|
|
refused
|
|
)
|
|
const sampledLogging = structuredClone(backend)
|
|
sampledLogging.change.after.log_config[0].sample_rate = RELAY_CELL_LOG_SAMPLE_RATE / 2
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, sampledLogging] },
|
|
sameCapConfig
|
|
),
|
|
refused
|
|
)
|
|
const disabledLogging = structuredClone(backend)
|
|
disabledLogging.change.after.log_config[0].enable = false
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, disabledLogging] },
|
|
sameCapConfig
|
|
),
|
|
refused
|
|
)
|
|
const replaced = structuredClone(backend)
|
|
replaced.change.actions = ['create', 'delete']
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager, replaced] }, sameCapConfig),
|
|
refused
|
|
)
|
|
// Only the same-cap wave targets a backend service; every other mode still refuses one.
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager, backend] },
|
|
{ ...config, mode: 'cell', image }
|
|
),
|
|
/only the exact instance template and MIG/
|
|
)
|
|
})
|