mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
c17's canary-apply failed closed at plan validation. Its instance template is from 2026-08-07 and predates the ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT line that every cell rolled since already carries, so the plan legitimately added it. The same-cap validator holds the whole startup script identical before and after except the image, and that line is not one it excluded, so the wave stopped with nothing applied. Pin the line for same-cap-cell exactly as bootstrap-cell already does, and exclude it from the before/after comparison. The cell may gain it; the pin is what refuses a roll that drops it or rewrites it to another identity. Both plan validations in the job now pass the capacity identity the job already requires. The same-cap contract is otherwise unchanged: any other stale line still fails closed, and needs a convergence apply before the cell can roll.
1057 lines
37 KiB
JavaScript
1057 lines
37 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { test } from 'node:test'
|
|
import {
|
|
parseCapacityPlanArguments,
|
|
validateCapacityPlan as validateCapacityPlanRaw
|
|
} from './validate-relay-capacity-plan.mjs'
|
|
|
|
const config = {
|
|
cellId: 'staging-gce-c3',
|
|
hardCap: 1_000,
|
|
unobservedBound: 60
|
|
}
|
|
|
|
function replacementConfiguration(references = [
|
|
'google_compute_instance_template.relay_gce_cell',
|
|
'each.key'
|
|
]) {
|
|
return {
|
|
root_module: {
|
|
resources: [{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell',
|
|
expressions: {
|
|
version: [{
|
|
instance_template: { references },
|
|
name: { constant_value: 'primary' }
|
|
}]
|
|
}
|
|
}]
|
|
}
|
|
}
|
|
}
|
|
|
|
function validateCapacityPlan(plan, planConfig) {
|
|
const replacement = plan.resource_changes.some(({ change }) =>
|
|
JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']))
|
|
return validateCapacityPlanRaw(
|
|
replacement && !plan.configuration
|
|
? { ...plan, configuration: replacementConfiguration() }
|
|
: plan,
|
|
planConfig
|
|
)
|
|
}
|
|
|
|
test('accepts only the exact canary template replacement and MIG update', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}`
|
|
const startupScript = (cap, bound, selectedImage, extra = '') =>
|
|
[
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
extra,
|
|
'docker run --detach \\',
|
|
' --name cloud-sql-proxy \\',
|
|
` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const script = startupScript(1_000, 60, image)
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startupScript(
|
|
600,
|
|
60,
|
|
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
|
|
)
|
|
},
|
|
after: { metadata_startup_script: script, self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const cellConfig = { ...config, mode: 'cell', image }
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 0
|
|
})
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
const wrongTemplateManager = structuredClone(manager)
|
|
wrongTemplateManager.change.after.version[0].instance_template =
|
|
'projects/project/global/instanceTemplates/unreviewed'
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, wrongTemplateManager] },
|
|
cellConfig
|
|
),
|
|
/does not bind the MIG/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlanRaw(
|
|
{
|
|
resource_changes: [template, manager],
|
|
configuration: replacementConfiguration([
|
|
'google_compute_instance_template.relay_gce_cell',
|
|
'var.unreviewed_key'
|
|
])
|
|
},
|
|
cellConfig
|
|
),
|
|
/reviewed template dependency/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlanRaw(
|
|
{ resource_changes: [template, manager] },
|
|
cellConfig
|
|
),
|
|
/reviewed template dependency/
|
|
)
|
|
const capacityServiceAccount =
|
|
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
|
|
const bootstrapTemplate = structuredClone(template)
|
|
const bootstrapManager = structuredClone(manager)
|
|
bootstrapTemplate.change.after.metadata_startup_script = [
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`,
|
|
script
|
|
].join('\n')
|
|
const bootstrapConfig = {
|
|
...cellConfig,
|
|
mode: 'bootstrap-cell',
|
|
capacityServiceAccount
|
|
}
|
|
const plannedValues = (startupScript) => ({
|
|
root_module: {
|
|
resources: [
|
|
{
|
|
address: bootstrapTemplate.address,
|
|
values: {
|
|
metadata_startup_script: startupScript,
|
|
self_link: 'projects/project/global/instanceTemplates/c26-reviewed'
|
|
}
|
|
},
|
|
{
|
|
address: bootstrapManager.address,
|
|
values: {
|
|
version: [{
|
|
instance_template:
|
|
'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed'
|
|
}]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
const managerOnly = structuredClone(bootstrapManager)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [managerOnly],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 1 }
|
|
)
|
|
const decoyPlannedValues = plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script.replaceAll(
|
|
image,
|
|
`us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}`
|
|
) + `\n# decoy '${image}'`
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly], planned_values: decoyPlannedValues },
|
|
bootstrapConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const obsoleteTemplate = structuredClone(bootstrapTemplate)
|
|
obsoleteTemplate.deposed = 'retired-template'
|
|
obsoleteTemplate.change.actions = ['delete']
|
|
obsoleteTemplate.change.after = null
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [managerOnly, obsoleteTemplate],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [obsoleteTemplate],
|
|
planned_values: plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
},
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 1 }
|
|
)
|
|
const wrongManagerReference = plannedValues(
|
|
bootstrapTemplate.change.after.metadata_startup_script
|
|
)
|
|
wrongManagerReference.root_module.resources[1].values.version[0].instance_template =
|
|
'projects/project/global/instanceTemplates/not-reviewed'
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly], planned_values: wrongManagerReference },
|
|
bootstrapConfig
|
|
),
|
|
/does not bind the MIG/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [managerOnly] },
|
|
bootstrapConfig
|
|
),
|
|
/no exact planned C26 state/
|
|
)
|
|
const restartedBootstrapManager = structuredClone(bootstrapManager)
|
|
restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }]
|
|
restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }]
|
|
restartedBootstrapManager.change.before.version[0].name =
|
|
'0/2026-08-10 23:30:14.196895+00:00'
|
|
restartedBootstrapManager.change.after.version[0].name = 'primary'
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, restartedBootstrapManager] },
|
|
bootstrapConfig
|
|
),
|
|
{ mode: 'bootstrap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [template, restartedBootstrapManager] },
|
|
cellConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestartManager = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestartManager.change.before.version[0].name = 'operator-version'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestartManager] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager)
|
|
unrecognizedPrimaryVersion.change.after.version[0].name = 'other'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager)
|
|
unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const missingRestartPolicy = structuredClone(restartedBootstrapManager)
|
|
delete missingRestartPolicy.change.before.update_policy
|
|
delete missingRestartPolicy.change.after.update_policy
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, missingRestartPolicy] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const missingRestartVersion = structuredClone(restartedBootstrapManager)
|
|
missingRestartVersion.change.before.version[0].name = 'primary'
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, missingRestartVersion] },
|
|
bootstrapConfig
|
|
),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
const duplicateHardCapTemplate = structuredClone(template)
|
|
duplicateHardCapTemplate.change.after.metadata_startup_script = [
|
|
script,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`
|
|
].join('\n')
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] },
|
|
cellConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const duplicateIdentityTemplate = structuredClone(bootstrapTemplate)
|
|
duplicateIdentityTemplate.change.after.metadata_startup_script = [
|
|
duplicateIdentityTemplate.change.after.metadata_startup_script,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'`
|
|
].join('\n')
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] },
|
|
bootstrapConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
{ ...bootstrapConfig, capacityServiceAccount: 'invalid' }
|
|
),
|
|
/invalid service account/
|
|
)
|
|
assert.throws(
|
|
() =>
|
|
validateCapacityPlan(
|
|
{ resource_changes: [bootstrapTemplate, bootstrapManager] },
|
|
cellConfig
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
manager.change.after.target_size = 0
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
manager.change.after.target_size = 1
|
|
template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad')
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
template.change.after.metadata_startup_script = startupScript(
|
|
1_000,
|
|
60,
|
|
image,
|
|
'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP='
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
template.change.after.metadata_startup_script = script
|
|
template.change.after_unknown = {
|
|
id: true,
|
|
self_link: true,
|
|
disk: [{ architecture: true }]
|
|
}
|
|
manager.change.after_unknown = {
|
|
fingerprint: true,
|
|
version: [{ instance_template: true }]
|
|
}
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
template.change.before.description = ''
|
|
template.change.after.description = null
|
|
template.change.before.disk = [{
|
|
architecture: '',
|
|
source_image: 'projects/cos-cloud/global/images/cos-stable-1'
|
|
}]
|
|
template.change.after.disk = [{
|
|
source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
|
|
}]
|
|
template.change.after_unknown.disk = [{ architecture: true }]
|
|
assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), {
|
|
mode: 'cell',
|
|
changes: 2
|
|
})
|
|
template.change.after.disk[0].source_image =
|
|
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different'
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
template.change.after.disk[0].source_image =
|
|
'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1'
|
|
manager.change.after_unknown = { target_size: true }
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig),
|
|
/outside the reviewed capacity fields/
|
|
)
|
|
})
|
|
|
|
test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
...(capacity === null
|
|
? []
|
|
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
|
...(trust ? [
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
] : []),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, trust: true }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const sameCapConfig = {
|
|
...config,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '1'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A pre-template-apply rollback resume validates drift for the image the
|
|
// cell already serves: the template leaves and re-enters the rollback image.
|
|
const resumeTemplate = structuredClone(template)
|
|
resumeTemplate.change.after.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
trust: true
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [resumeTemplate, manager] },
|
|
{ ...sameCapConfig, image: rollbackImage }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const asiaTemplate = structuredClone(template)
|
|
const asiaManager = structuredClone(manager)
|
|
asiaTemplate.address =
|
|
'google_compute_instance_template.relay_gce_cell["production-gce-c28"]'
|
|
asiaManager.address =
|
|
'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]'
|
|
asiaTemplate.change.before.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
cap: 3_000
|
|
})
|
|
asiaTemplate.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
cap: 3_000,
|
|
trust: true
|
|
})
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [asiaTemplate, asiaManager] },
|
|
{ ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const changedCap = structuredClone(template)
|
|
changedCap.change.before.metadata_startup_script = startup({
|
|
selectedImage: rollbackImage,
|
|
cap: 600
|
|
})
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...sameCapConfig, rollbackImage: image }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const wrongTrust = structuredClone(template)
|
|
wrongTrust.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
trust: true
|
|
}).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com')
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
|
|
const imageOnly = structuredClone(template)
|
|
imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image })
|
|
const imageOnlyConfig = {
|
|
...config,
|
|
mode: 'same-cap-image',
|
|
image,
|
|
rollbackImage
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig),
|
|
{ mode: 'same-cap-image', changes: 2, changeKind: 'replacement' }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager] },
|
|
{ ...imageOnlyConfig, rollbackImage: image }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
const changedTrust = structuredClone(imageOnly)
|
|
changedTrust.change.after.metadata_startup_script +=
|
|
`\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
const plannedValues = {
|
|
root_module: {
|
|
resources: [
|
|
{
|
|
address: imageOnly.address,
|
|
values: {
|
|
metadata_startup_script: imageOnly.change.after.metadata_startup_script,
|
|
self_link: 'projects/project/global/instanceTemplates/target'
|
|
}
|
|
},
|
|
{
|
|
address: manager.address,
|
|
values: {
|
|
version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
const obsoleteTemplate = structuredClone(imageOnly)
|
|
obsoleteTemplate.deposed = 'obsolete'
|
|
obsoleteTemplate.change.actions = ['delete']
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [obsoleteTemplate], planned_values: plannedValues },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' }
|
|
)
|
|
const anotherObsoleteTemplate = {
|
|
...structuredClone(obsoleteTemplate),
|
|
deposed: 'another-obsolete'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [obsoleteTemplate, anotherObsoleteTemplate],
|
|
planned_values: plannedValues
|
|
},
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{
|
|
resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate],
|
|
planned_values: plannedValues
|
|
},
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' }
|
|
)
|
|
const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate)
|
|
invalidObsoleteTemplate.change.actions = ['update']
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] },
|
|
imageOnlyConfig
|
|
),
|
|
/change only the exact instance template and MIG/
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [manager], planned_values: plannedValues },
|
|
imageOnlyConfig
|
|
),
|
|
{ mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' }
|
|
)
|
|
})
|
|
|
|
test('protocol-0 same-cap cells roll without rehome trust lines', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, trust = false }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
|
|
...(trust ? [
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
|
] : []),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const template = {
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
|
|
after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null },
|
|
after_unknown: { self_link: true }
|
|
}
|
|
}
|
|
const manager = {
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
const asiaConfig = {
|
|
cellId: 'production-gce-c27',
|
|
hardCap: 3_000,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
assert.deepEqual(
|
|
validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
const gainsTrust = structuredClone(template)
|
|
gainsTrust.change.after.metadata_startup_script = startup({
|
|
selectedImage: image,
|
|
trust: true
|
|
})
|
|
assert.throws(
|
|
() => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// Under protocol 1 that same script is the reviewed roll: trust is added, not drift.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(
|
|
{ resource_changes: [gainsTrust, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: '1' }
|
|
),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
// A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged.
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: '1' }
|
|
),
|
|
/reviewed image and capacity/
|
|
)
|
|
for (const protocol of [undefined, '', '2', 'yes']) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(
|
|
{ resource_changes: [template, manager] },
|
|
{ ...asiaConfig, regionalRehomeProtocol: protocol }
|
|
),
|
|
/invalid regional rehome protocol/
|
|
)
|
|
}
|
|
})
|
|
|
|
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, capacity }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
...(capacity === null
|
|
? []
|
|
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const plan = (beforeCapacity, afterCapacity) => ({
|
|
resource_changes: [
|
|
{
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startup({
|
|
selectedImage: rollbackImage,
|
|
capacity: beforeCapacity
|
|
})
|
|
},
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
},
|
|
{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
]
|
|
})
|
|
const config = {
|
|
cellId: 'production-gce-c17',
|
|
hardCap: 600,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '0'
|
|
}
|
|
// A template old enough to predate the line gains it, which is the only move allowed.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(plan(null, capacityIdentity), config),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.deepEqual(
|
|
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
for (const [before, after] of [
|
|
[capacityIdentity, null],
|
|
[null, null],
|
|
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
|
|
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
|
|
]) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(before, after), config),
|
|
/reviewed image and capacity/,
|
|
`${before} -> ${after}`
|
|
)
|
|
}
|
|
// Without the pin there is nothing reviewing the line the comparison now ignores.
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
|
...config,
|
|
capacityServiceAccount: undefined
|
|
}),
|
|
/invalid service account/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
|
...config,
|
|
capacityServiceAccount: 'not-an-email'
|
|
}),
|
|
/invalid service account/
|
|
)
|
|
})
|
|
|
|
test('the capacity identity argument is required by same-cap-cell mode', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const base = [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c17',
|
|
'--hard-cap', '600',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
'--regional-rehome-protocol', '0'
|
|
]
|
|
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
|
|
/--capacity-service-account is invalid/
|
|
)
|
|
assert.equal(
|
|
parseCapacityPlanArguments([
|
|
...base,
|
|
'--capacity-service-account',
|
|
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
]).capacityServiceAccount,
|
|
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
)
|
|
})
|
|
|
|
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const sameCapArguments = (...extra) => [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c27',
|
|
'--hard-cap', '3000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
...extra
|
|
]
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0'))
|
|
.regionalRehomeProtocol,
|
|
'0'
|
|
)
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
|
|
'3'
|
|
)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments(sameCapArguments()),
|
|
/requires rollback image and rehome trust config/
|
|
)
|
|
for (const protocol of ['', '2', 'true']) {
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)),
|
|
/requires rollback image and rehome trust config/
|
|
)
|
|
}
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([
|
|
'--mode', 'bootstrap-cell',
|
|
'--cell-id', 'staging-gce-c3',
|
|
'--hard-cap', '1000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
|
|
'--regional-rehome-protocol', '0'
|
|
]),
|
|
/applies only to same-cap-cell validation/
|
|
)
|
|
})
|
|
|
|
test('the reviewed database pool is pinned for the cells that emit one', () => {
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
|
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
|
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
|
const startup = ({ selectedImage, pool }) => [
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
|
...(pool === undefined
|
|
? []
|
|
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
|
|
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
|
`docker pull '${selectedImage}'`,
|
|
'docker run --detach \\',
|
|
' --name orca-relay \\',
|
|
` '${selectedImage}'`
|
|
].join('\n')
|
|
const rollPlan = (before, after) => ({
|
|
resource_changes: [
|
|
{
|
|
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['create', 'delete'],
|
|
before: {
|
|
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
|
|
},
|
|
after: {
|
|
metadata_startup_script: startup({ selectedImage: image, pool: after }),
|
|
self_link: null
|
|
},
|
|
after_unknown: { self_link: true }
|
|
}
|
|
},
|
|
{
|
|
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
|
|
change: {
|
|
actions: ['update'],
|
|
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
|
after: { target_size: 1, version: [{ instance_template: null }] },
|
|
after_unknown: { version: [{ instance_template: true }] }
|
|
}
|
|
}
|
|
]
|
|
})
|
|
const asiaConfig = {
|
|
cellId: 'production-gce-c27',
|
|
hardCap: 3_000,
|
|
unobservedBound: 60,
|
|
mode: 'same-cap-cell',
|
|
image,
|
|
rollbackImage,
|
|
capacityServiceAccount: capacityIdentity,
|
|
rehomeDirectorServiceAccount: directorIdentity,
|
|
rehomeAudience: audience,
|
|
regionalRehomeProtocol: '1'
|
|
}
|
|
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A cell on the root pool default emits no line at all, and gaining one is real drift.
|
|
assert.deepEqual(
|
|
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
|
|
{ mode: 'same-cap-cell', changes: 2 }
|
|
)
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A line already on the live template is still unreviewed without the pin, even standing still.
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
|
|
/reviewed image and capacity/
|
|
)
|
|
// A pin must also fail closed when the plan drops the line it names.
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
|
|
/reviewed image and capacity/
|
|
)
|
|
for (const pool of ['', '0', '101', 'ten']) {
|
|
assert.throws(
|
|
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
|
|
/invalid database pool max/
|
|
)
|
|
}
|
|
})
|
|
|
|
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
|
|
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
|
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
|
const sameCapArguments = (...extra) => [
|
|
'--mode', 'same-cap-cell',
|
|
'--cell-id', 'production-gce-c27',
|
|
'--hard-cap', '3000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--rollback-image', rollbackImage,
|
|
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
|
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
|
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
|
'--regional-rehome-protocol', '1',
|
|
...extra
|
|
]
|
|
assert.equal(
|
|
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
|
|
'16'
|
|
)
|
|
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
|
|
assert.throws(
|
|
() => parseCapacityPlanArguments([
|
|
'--mode', 'bootstrap-cell',
|
|
'--cell-id', 'staging-gce-c3',
|
|
'--hard-cap', '1000',
|
|
'--unobserved-bound', '60',
|
|
'--image', image,
|
|
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
|
|
'--database-pool-max', '16'
|
|
]),
|
|
/applies only to same-cap-cell validation/
|
|
)
|
|
})
|