mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
fix(cloud): pin the asia cell database pool in the same-cap plan validator (#21171)
* fix(cloud): pin the asia cell database pool in the same-cap plan validator Raising `database_pool_max` from 10 to 16 for production-gce-c27, c28 and c29 made every same-cap roll of those three cells fail closed at plan validation. The cell startup template emits `ORCA_RELAY_DATABASE_POOL_MAX` only for a cell whose region differs from the root region or whose pool is off the default, so the asia cells carry that line while the us-central1 cells do not. The plan validator requires the before and after startup scripts to normalize to the same text, masking only the lines it independently pins to a reviewed value. The pool line was neither masked nor pinned, so the live template's `'10'` and the plan's `'16'` were read as unreviewed drift. The validator gains an optional `--database-pool-max`, accepted in `same-cap-cell` mode alone. When it is supplied the after-script must contain exactly that pool line and the line is masked from the equality check; when it is not supplied the after-script must contain no pool line at all. Masking without the pin would have removed the guard rather than moved it. The same-cap job resolves the expected pool next to the hard cap, cross-checks it against the committed `relay_gce_cells` map (asserting the default 10 for the us-central1 cells), and passes the flag to both validator invocations only for the cells that emit the line. * test(cloud): require the pool pin for a line the live template already carries
This commit is contained in:
@@ -213,10 +213,12 @@ jobs:
|
||||
c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26)
|
||||
EXPECTED_HARD_CAP=1000
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c27|c28|c29)
|
||||
EXPECTED_HARD_CAP=3000
|
||||
EXPECTED_REGION=asia-east2
|
||||
EXPECTED_DATABASE_POOL_MAX=16
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
@@ -236,6 +238,10 @@ jobs:
|
||||
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
|
||||
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
|
||||
"${EXPECTED_UNOBSERVED_BOUND}"
|
||||
# The startup script emits a pool line only off the root default, so an unpinned cell
|
||||
# must still be on that default or its plan would carry a line nothing reviews.
|
||||
test "$(jq -r '.database_pool_max' <<< "${CURRENT_SHAPE}")" = \
|
||||
"${EXPECTED_DATABASE_POOL_MAX:-10}"
|
||||
TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")"
|
||||
MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}"
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
@@ -264,6 +270,7 @@ jobs:
|
||||
echo "MIG_NAME=${MIG_NAME}"
|
||||
echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}"
|
||||
echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}"
|
||||
echo "EXPECTED_DATABASE_POOL_MAX=${EXPECTED_DATABASE_POOL_MAX}"
|
||||
echo "EXPECTED_REGION=${EXPECTED_REGION}"
|
||||
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
|
||||
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
|
||||
@@ -461,6 +468,11 @@ jobs:
|
||||
env:
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
POOL_ARGUMENTS=()
|
||||
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
|
||||
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
|
||||
fi
|
||||
# Zero resource changes prove the prior run's apply completed and no
|
||||
# restart will follow, keeping the incarnation check honest. Root
|
||||
# outputs may lag a targeted apply, so judge resource_changes only.
|
||||
@@ -502,6 +514,7 @@ jobs:
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
"${POOL_ARGUMENTS[@]}" \
|
||||
| jq -e '.changes == 2' >/dev/null
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
@@ -514,6 +527,11 @@ jobs:
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
POOL_ARGUMENTS=()
|
||||
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
|
||||
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
|
||||
fi
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
|
||||
@@ -528,7 +546,8 @@ jobs:
|
||||
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
"${POOL_ARGUMENTS[@]}"
|
||||
terraform -chdir=infra/terraform apply -auto-approve \
|
||||
"${RUNNER_TEMP}/relay-same-cap.tfplan"
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
|
||||
@@ -75,7 +75,7 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/
|
||||
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {16}"\$\{POOL_ARGUMENTS\[@\]\}" \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/
|
||||
)
|
||||
assert.match(job, /resume requires the isolated migration-only cell/)
|
||||
assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
|
||||
|
||||
@@ -31,10 +31,21 @@ function rehomeSourceCells() {
|
||||
)
|
||||
}
|
||||
|
||||
function startupScript({ cap, image, trusted }) {
|
||||
// The job cross-checks its pinned pool against the committed map; model the same read.
|
||||
function tfvarsDatabasePoolMax(cellId) {
|
||||
const start = production.indexOf(`"${cellId}" = {`)
|
||||
assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`)
|
||||
const block = production.slice(start, production.indexOf('\n }', start))
|
||||
return /database_pool_max\s*=\s*(\d+)/.exec(block)?.[1] ?? '10'
|
||||
}
|
||||
|
||||
function startupScript({ cap, image, trusted, pool }) {
|
||||
return [
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
||||
...(pool === undefined
|
||||
? []
|
||||
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
||||
...(trusted ? [
|
||||
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${DIRECTOR_IDENTITY}'`,
|
||||
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${AUDIENCE}'`
|
||||
@@ -48,7 +59,7 @@ function startupScript({ cap, image, trusted }) {
|
||||
}
|
||||
|
||||
// The exact shape the apply step's plan has: template replaced, MIG rebound to it.
|
||||
function rollPlan({ cellId, cap, protocol }) {
|
||||
function rollPlan({ cellId, cap, protocol, pool }) {
|
||||
return {
|
||||
configuration: {
|
||||
root_module: {
|
||||
@@ -77,14 +88,17 @@ function rollPlan({ cellId, cap, protocol }) {
|
||||
metadata_startup_script: startupScript({
|
||||
cap,
|
||||
image: ROLLBACK_IMAGE,
|
||||
trusted: protocol >= 1
|
||||
trusted: protocol >= 1,
|
||||
// The live template predates the reviewed pool raise, as every asia cell's does.
|
||||
pool: pool === undefined ? undefined : '10'
|
||||
})
|
||||
},
|
||||
after: {
|
||||
metadata_startup_script: startupScript({
|
||||
cap,
|
||||
image: TARGET_IMAGE,
|
||||
trusted: protocol >= 1
|
||||
trusted: protocol >= 1,
|
||||
pool
|
||||
}),
|
||||
self_link: null
|
||||
},
|
||||
@@ -108,7 +122,8 @@ function hostname(cellId) {
|
||||
return cellId.slice('production-gce-'.length)
|
||||
}
|
||||
|
||||
// The job resolves cap and region from the cell id before any admin call; run that block alone.
|
||||
// The job resolves cap, region, and pool from the cell id before any admin call; run that block
|
||||
// alone. An empty pool is the root default, which the startup template emits no line for.
|
||||
function resolveCellShape(cellId) {
|
||||
const start = workflow.indexOf(' TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"')
|
||||
assert.notEqual(start, -1, 'the same-cap cell shape block is missing')
|
||||
@@ -119,10 +134,17 @@ function resolveCellShape(cellId) {
|
||||
'-euo',
|
||||
'pipefail',
|
||||
'-c',
|
||||
`${script}\necho "\${EXPECTED_REGION} \${EXPECTED_HARD_CAP}"`
|
||||
`${script}\necho "\${EXPECTED_REGION} \${EXPECTED_HARD_CAP} pool=\${EXPECTED_DATABASE_POOL_MAX}"`
|
||||
], { env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
|
||||
}
|
||||
|
||||
function cellShape(cellId) {
|
||||
const resolved = resolveCellShape(cellId)
|
||||
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
|
||||
const [, cap, pool] = resolved.stdout.trim().split(' ')
|
||||
return { cap: Number(cap), pool: pool.slice('pool='.length) || undefined }
|
||||
}
|
||||
|
||||
describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
it('parses every wave cell through the same-cap canary allowlist', () => {
|
||||
for (const cellId of SAME_CAP_CELLS) {
|
||||
@@ -143,11 +165,16 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('resolves a cap and region for every wave cell and refuses anything else', () => {
|
||||
it('resolves a cap, region, and pool for every wave cell and refuses anything else', () => {
|
||||
for (const cellId of SAME_CAP_CELLS) {
|
||||
const resolved = resolveCellShape(cellId)
|
||||
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
|
||||
assert.match(resolved.stdout.trim(), /^(us-central1 1000|asia-east2 3000)$/)
|
||||
assert.match(
|
||||
resolved.stdout.trim(),
|
||||
/^(us-central1 1000 pool=|asia-east2 3000 pool=16)$/,
|
||||
cellId
|
||||
)
|
||||
assert.equal(tfvarsDatabasePoolMax(cellId), cellShape(cellId).pool ?? '10', cellId)
|
||||
}
|
||||
assert.equal(resolveCellShape('production-gce-c17').status, 1)
|
||||
assert.equal(resolveCellShape('production-gce-c30').status, 1)
|
||||
@@ -165,7 +192,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('passes this cell\'s rehome protocol on every plan validation the job runs', () => {
|
||||
it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {
|
||||
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
|
||||
assert.equal(invocations.length, 2)
|
||||
for (const invocation of invocations) {
|
||||
@@ -174,25 +201,34 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
const call = lines.slice(0, end + 1).join(' ')
|
||||
assert.match(call, /--mode same-cap-cell/)
|
||||
assert.match(call, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
|
||||
assert.match(call, /"\$\{POOL_ARGUMENTS\[@\]\}"/)
|
||||
}
|
||||
// Each of those steps must build the flag from the resolved pool, and only when there is one.
|
||||
const builders = workflow.split(
|
||||
'if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then\n' +
|
||||
' POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")'
|
||||
)
|
||||
assert.equal(builders.length, 3)
|
||||
assert.equal(workflow.split('POOL_ARGUMENTS=()').length, 3)
|
||||
})
|
||||
|
||||
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
|
||||
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
|
||||
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
|
||||
const { cap, pool } = cellShape(cellId)
|
||||
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
|
||||
const config = {
|
||||
mode: 'same-cap-cell',
|
||||
cellId,
|
||||
hardCap: Number(cap),
|
||||
hardCap: cap,
|
||||
unobservedBound: 60,
|
||||
image: TARGET_IMAGE,
|
||||
rollbackImage: ROLLBACK_IMAGE,
|
||||
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
||||
rehomeAudience: AUDIENCE,
|
||||
regionalRehomeProtocol: String(protocol)
|
||||
regionalRehomeProtocol: String(protocol),
|
||||
databasePoolMax: pool
|
||||
}
|
||||
const plan = rollPlan({ cellId, cap, protocol })
|
||||
const plan = rollPlan({ cellId, cap, protocol, pool })
|
||||
assert.deepEqual(
|
||||
validateCapacityPlan(plan, config),
|
||||
{ mode: 'same-cap-cell', changes: 2 },
|
||||
@@ -207,6 +243,15 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
/reviewed image and capacity/,
|
||||
cellId
|
||||
)
|
||||
// Dropping the pin must reject a pinned cell, and adding one must reject a default cell.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(plan, {
|
||||
...config,
|
||||
databasePoolMax: pool === undefined ? '16' : undefined
|
||||
}),
|
||||
/reviewed image and capacity/,
|
||||
cellId
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ const SERVICE_ACCOUNT_EMAIL =
|
||||
const REHOME_CONFIG =
|
||||
/^ printf 'ORCA_RELAY_REHOME_(?:DIRECTOR_SERVICE_ACCOUNT|AUDIENCE)=%s\\n' '[^'\n]+'$/
|
||||
|
||||
const DATABASE_POOL_MAX = /^ printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '[0-9]+'$/
|
||||
|
||||
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
|
||||
function rehomeProtocol({ regionalRehomeProtocol }) {
|
||||
if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) {
|
||||
@@ -15,6 +17,16 @@ function rehomeProtocol({ regionalRehomeProtocol }) {
|
||||
return Number(regionalRehomeProtocol)
|
||||
}
|
||||
|
||||
// Only cells off the root pool default get a pool line, so the caller states whether to expect one.
|
||||
function databasePoolMax({ databasePoolMax: value }) {
|
||||
if (value === undefined) return undefined
|
||||
const pool = Number(value)
|
||||
if (!/^[0-9]+$/.test(String(value)) || pool < 1 || pool > 100) {
|
||||
throw new Error('same-cap Terraform plan has an invalid database pool max')
|
||||
}
|
||||
return String(pool)
|
||||
}
|
||||
|
||||
export function parseCapacityPlanArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
@@ -48,6 +60,9 @@ export function parseCapacityPlanArguments(argv) {
|
||||
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
|
||||
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
|
||||
}
|
||||
if (values.mode !== 'same-cap-cell' && values['database-pool-max'] !== undefined) {
|
||||
throw new Error('--database-pool-max applies only to same-cap-cell validation')
|
||||
}
|
||||
if (values.mode === 'same-cap-image' && !values['rollback-image']) {
|
||||
throw new Error('same-cap image validation requires a rollback image')
|
||||
}
|
||||
@@ -67,7 +82,8 @@ export function parseCapacityPlanArguments(argv) {
|
||||
rollbackImage: values['rollback-image'],
|
||||
rehomeDirectorServiceAccount: values['rehome-director-service-account'],
|
||||
rehomeAudience: values['rehome-audience'],
|
||||
regionalRehomeProtocol: values['regional-rehome-protocol']
|
||||
regionalRehomeProtocol: values['regional-rehome-protocol'],
|
||||
databasePoolMax: values['database-pool-max']
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,7 +198,8 @@ function normalizedStartupScript(
|
||||
script,
|
||||
stripCapacityIdentity = false,
|
||||
stripRehomeConfig = false,
|
||||
preserveCapacity = false
|
||||
preserveCapacity = false,
|
||||
stripDatabasePoolMax = false
|
||||
) {
|
||||
const image = relayImage(script)
|
||||
if (!image) throw new Error('cell plan startup script has no Relay image')
|
||||
@@ -197,7 +214,8 @@ function normalizedStartupScript(
|
||||
(line) =>
|
||||
(preserveCapacity || !capacityAssignment.test(line)) &&
|
||||
(!stripCapacityIdentity || !capacityIdentity.test(line)) &&
|
||||
(!stripRehomeConfig || !REHOME_CONFIG.test(line))
|
||||
(!stripRehomeConfig || !REHOME_CONFIG.test(line)) &&
|
||||
(!stripDatabasePoolMax || !DATABASE_POOL_MAX.test(line))
|
||||
)
|
||||
.join('\n')
|
||||
.replaceAll(image, '<relay-image>')
|
||||
@@ -245,10 +263,23 @@ function requireDesiredStartupScript(script, config) {
|
||||
config.mode === 'same-cap-cell' &&
|
||||
!rehomeTrusted &&
|
||||
lines.some((line) => REHOME_CONFIG.test(line))
|
||||
const pool = config.mode === 'same-cap-cell' ? databasePoolMax(config) : undefined
|
||||
if (pool !== undefined) {
|
||||
expected.push([
|
||||
DATABASE_POOL_MAX,
|
||||
` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`
|
||||
])
|
||||
}
|
||||
// An unpinned cell sits on the root pool default, so gaining a pool line is real drift.
|
||||
const unexpectedDatabasePoolMax =
|
||||
config.mode === 'same-cap-cell' &&
|
||||
pool === undefined &&
|
||||
lines.some((line) => DATABASE_POOL_MAX.test(line))
|
||||
if (
|
||||
typeof script !== 'string' ||
|
||||
relayImage(script) !== config.image ||
|
||||
unexpectedRehome ||
|
||||
unexpectedDatabasePoolMax ||
|
||||
expected.some(([pattern, line]) => !hasExactSingleAssignment(lines, pattern, line))
|
||||
) {
|
||||
throw new Error('cell plan does not contain the reviewed image and capacity')
|
||||
@@ -421,6 +452,8 @@ function cellPlan(plan, changes, config) {
|
||||
const script = template.change.after?.metadata_startup_script
|
||||
requireDesiredStartupScript(script, config)
|
||||
const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode)
|
||||
// Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly.
|
||||
const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined
|
||||
if (
|
||||
typeof beforeScript !== 'string' ||
|
||||
(sameCap && relayImage(beforeScript) !== config.rollbackImage) ||
|
||||
@@ -428,12 +461,14 @@ function cellPlan(plan, changes, config) {
|
||||
beforeScript,
|
||||
config.mode === 'bootstrap-cell',
|
||||
config.mode === 'same-cap-cell',
|
||||
sameCap
|
||||
sameCap,
|
||||
stripPool
|
||||
) !== normalizedStartupScript(
|
||||
script,
|
||||
config.mode === 'bootstrap-cell',
|
||||
config.mode === 'same-cap-cell',
|
||||
sameCap
|
||||
sameCap,
|
||||
stripPool
|
||||
)
|
||||
) {
|
||||
throw new Error('cell plan does not contain the reviewed image and capacity')
|
||||
@@ -473,6 +508,7 @@ export function validateCapacityPlan(plan, config) {
|
||||
}
|
||||
if (config.mode === 'same-cap-cell') {
|
||||
rehomeProtocol(config)
|
||||
databasePoolMax(config)
|
||||
}
|
||||
if (
|
||||
config.mode === 'same-cap-cell' &&
|
||||
|
||||
@@ -783,3 +783,134 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
|
||||
/applies only to same-cap-cell validation/
|
||||
)
|
||||
})
|
||||
|
||||
test('the reviewed database pool is pinned for the cells that emit one', () => {
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
||||
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
||||
const startup = ({ selectedImage, pool }) => [
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
||||
...(pool === undefined
|
||||
? []
|
||||
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
||||
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
||||
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
|
||||
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
||||
`docker pull '${selectedImage}'`,
|
||||
'docker run --detach \\',
|
||||
' --name orca-relay \\',
|
||||
` '${selectedImage}'`
|
||||
].join('\n')
|
||||
const rollPlan = (before, after) => ({
|
||||
resource_changes: [
|
||||
{
|
||||
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
|
||||
change: {
|
||||
actions: ['create', 'delete'],
|
||||
before: {
|
||||
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
|
||||
},
|
||||
after: {
|
||||
metadata_startup_script: startup({ selectedImage: image, pool: after }),
|
||||
self_link: null
|
||||
},
|
||||
after_unknown: { self_link: true }
|
||||
}
|
||||
},
|
||||
{
|
||||
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
|
||||
change: {
|
||||
actions: ['update'],
|
||||
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
||||
after: { target_size: 1, version: [{ instance_template: null }] },
|
||||
after_unknown: { version: [{ instance_template: true }] }
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
const asiaConfig = {
|
||||
cellId: 'production-gce-c27',
|
||||
hardCap: 3_000,
|
||||
unobservedBound: 60,
|
||||
mode: 'same-cap-cell',
|
||||
image,
|
||||
rollbackImage,
|
||||
rehomeDirectorServiceAccount: directorIdentity,
|
||||
rehomeAudience: audience,
|
||||
regionalRehomeProtocol: '1'
|
||||
}
|
||||
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
|
||||
assert.deepEqual(
|
||||
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
|
||||
{ mode: 'same-cap-cell', changes: 2 }
|
||||
)
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
// A cell on the root pool default emits no line at all, and gaining one is real drift.
|
||||
assert.deepEqual(
|
||||
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
|
||||
{ mode: 'same-cap-cell', changes: 2 }
|
||||
)
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
// A line already on the live template is still unreviewed without the pin, even standing still.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
// A pin must also fail closed when the plan drops the line it names.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
for (const pool of ['', '0', '101', 'ten']) {
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
|
||||
/invalid database pool max/
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
const sameCapArguments = (...extra) => [
|
||||
'--mode', 'same-cap-cell',
|
||||
'--cell-id', 'production-gce-c27',
|
||||
'--hard-cap', '3000',
|
||||
'--unobserved-bound', '60',
|
||||
'--image', image,
|
||||
'--rollback-image', rollbackImage,
|
||||
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
||||
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
||||
'--regional-rehome-protocol', '1',
|
||||
...extra
|
||||
]
|
||||
assert.equal(
|
||||
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
|
||||
'16'
|
||||
)
|
||||
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
|
||||
assert.throws(
|
||||
() => parseCapacityPlanArguments([
|
||||
'--mode', 'bootstrap-cell',
|
||||
'--cell-id', 'staging-gce-c3',
|
||||
'--hard-cap', '1000',
|
||||
'--unobserved-bound', '60',
|
||||
'--image', image,
|
||||
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
|
||||
'--database-pool-max', '16'
|
||||
]),
|
||||
/applies only to same-cap-cell validation/
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user