mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
fix(relay-ops): pin the capacity identity so a stale same-cap template can roll (#21314)
c17's canary-apply failed closed at plan validation. Its instance template is from 2026-08-07 and predates the ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT line that every cell rolled since already carries, so the plan legitimately added it. The same-cap validator holds the whole startup script identical before and after except the image, and that line is not one it excluded, so the wave stopped with nothing applied. Pin the line for same-cap-cell exactly as bootstrap-cell already does, and exclude it from the before/after comparison. The cell may gain it; the pin is what refuses a roll that drops it or rewrites it to another identity. Both plan validations in the job now pass the capacity identity the job already requires. The same-cap contract is otherwise unchanged: any other stale line still fails closed, and needs a convergence apply before the cell can roll.
This commit is contained in:
@@ -541,6 +541,7 @@ jobs:
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
@@ -586,6 +587,7 @@ jobs:
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${DESIRED_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
@@ -619,6 +621,7 @@ jobs:
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
|
||||
@@ -67,11 +67,11 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
|
||||
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,300}CAPACITY_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT \}\}\n {10}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
|
||||
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
|
||||
@@ -20,6 +20,7 @@ const production = readFileSync(
|
||||
)
|
||||
const REHOME_SOURCE_CELLS = rehomeSourceCells()
|
||||
const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
|
||||
const CAPACITY_IDENTITY = 'orca-cloud-gha-cap@onorca-cloud.iam.gserviceaccount.com'
|
||||
const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain'
|
||||
const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}`
|
||||
const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}`
|
||||
@@ -53,10 +54,13 @@ function tfvarsCellBlock(cellId) {
|
||||
return production.slice(start, production.indexOf('\n }', start))
|
||||
}
|
||||
|
||||
function startupScript({ cap, image, trusted, pool }) {
|
||||
function startupScript({ cap, image, trusted, pool, capacityIdentity = CAPACITY_IDENTITY }) {
|
||||
return [
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
||||
...(capacityIdentity === null
|
||||
? []
|
||||
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`]),
|
||||
...(pool === undefined
|
||||
? []
|
||||
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
||||
@@ -73,7 +77,11 @@ function startupScript({ cap, image, trusted, pool }) {
|
||||
}
|
||||
|
||||
// The exact shape the apply step's plan has: template replaced, MIG rebound to it.
|
||||
function rollPlan({ cellId, cap, protocol, pool }) {
|
||||
function rollPlan({
|
||||
cellId, cap, protocol, pool,
|
||||
beforeCapacityIdentity = CAPACITY_IDENTITY,
|
||||
afterCapacityIdentity = CAPACITY_IDENTITY
|
||||
}) {
|
||||
return {
|
||||
configuration: {
|
||||
root_module: {
|
||||
@@ -104,7 +112,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
|
||||
image: ROLLBACK_IMAGE,
|
||||
trusted: protocol >= 1,
|
||||
// The live template predates the reviewed pool raise, as every asia cell's does.
|
||||
pool: pool === undefined ? undefined : '10'
|
||||
pool: pool === undefined ? undefined : '10',
|
||||
capacityIdentity: beforeCapacityIdentity
|
||||
})
|
||||
},
|
||||
after: {
|
||||
@@ -112,7 +121,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
|
||||
cap,
|
||||
image: TARGET_IMAGE,
|
||||
trusted: protocol >= 1,
|
||||
pool
|
||||
pool,
|
||||
capacityIdentity: afterCapacityIdentity
|
||||
}),
|
||||
self_link: null
|
||||
},
|
||||
@@ -298,6 +308,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
unobservedBound: 60,
|
||||
image: TARGET_IMAGE,
|
||||
rollbackImage: ROLLBACK_IMAGE,
|
||||
capacityServiceAccount: CAPACITY_IDENTITY,
|
||||
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
||||
rehomeAudience: AUDIENCE,
|
||||
regionalRehomeProtocol: String(protocol),
|
||||
@@ -340,6 +351,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
unobservedBound: 60,
|
||||
image: TARGET_IMAGE,
|
||||
rollbackImage: ROLLBACK_IMAGE,
|
||||
capacityServiceAccount: CAPACITY_IDENTITY,
|
||||
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
||||
rehomeAudience: AUDIENCE,
|
||||
regionalRehomeProtocol: '0'
|
||||
@@ -443,6 +455,67 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('rolls a template stale enough to predate the pinned capacity identity', () => {
|
||||
// Exactly c17's shape on 2026-09-18: its live template is from 2026-08-07 and has no
|
||||
// capacity identity line, so the roll adds one. Run 35290908836 failed closed here.
|
||||
const cellId = 'production-gce-c17'
|
||||
const config = {
|
||||
mode: 'same-cap-cell',
|
||||
cellId,
|
||||
hardCap: 600,
|
||||
unobservedBound: 60,
|
||||
image: TARGET_IMAGE,
|
||||
rollbackImage: ROLLBACK_IMAGE,
|
||||
capacityServiceAccount: CAPACITY_IDENTITY,
|
||||
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
||||
rehomeAudience: AUDIENCE,
|
||||
regionalRehomeProtocol: '0'
|
||||
}
|
||||
const stale = rollPlan({ cellId, cap: 600, protocol: 0, beforeCapacityIdentity: null })
|
||||
assert.deepEqual(validateCapacityPlan(stale, config), { mode: 'same-cap-cell', changes: 2 })
|
||||
// The line may only be gained. A roll may not rewrite it,
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(stale, {
|
||||
...config,
|
||||
capacityServiceAccount: 'orca-cloud-gha-other@onorca-cloud.iam.gserviceaccount.com'
|
||||
}),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
// nor drop it from a template that already carries one.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(
|
||||
rollPlan({ cellId, cap: 600, protocol: 0, afterCapacityIdentity: null }),
|
||||
config
|
||||
),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
// A same-cap roll cannot run without the identity pinned at all.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(stale, { ...config, capacityServiceAccount: undefined }),
|
||||
/invalid service account/
|
||||
)
|
||||
})
|
||||
|
||||
it('pins the capacity identity on every plan validation the job runs', () => {
|
||||
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
|
||||
assert.equal(invocations.length, 2)
|
||||
for (const invocation of invocations) {
|
||||
const lines = invocation.split('\n')
|
||||
const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
|
||||
assert.match(
|
||||
lines.slice(0, end + 1).join(' '),
|
||||
/--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/
|
||||
)
|
||||
}
|
||||
// Both steps must read it from the same repository variable the job already requires.
|
||||
assert.equal(
|
||||
workflow.split(
|
||||
'CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}'
|
||||
).length,
|
||||
4
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves the US-only capacity job on the default allowlist', () => {
|
||||
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
|
||||
})
|
||||
|
||||
@@ -47,7 +47,10 @@ export function parseCapacityPlanArguments(argv) {
|
||||
return value
|
||||
}
|
||||
if (!values.image) throw new Error('missing --image')
|
||||
if (values.mode === 'bootstrap-cell' && !values['capacity-service-account']) {
|
||||
if (
|
||||
['bootstrap-cell', 'same-cap-cell'].includes(values.mode) &&
|
||||
!values['capacity-service-account']
|
||||
) {
|
||||
throw new Error('missing --capacity-service-account')
|
||||
}
|
||||
if (
|
||||
@@ -239,7 +242,10 @@ function requireDesiredStartupScript(script, config) {
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'`
|
||||
]
|
||||
]
|
||||
if (config.mode === 'bootstrap-cell') {
|
||||
// A same-cap cell whose template predates this line gains it on its next roll, so the
|
||||
// before/after comparison ignores it; pinning the exact identity here is what reviews it,
|
||||
// and what stops a roll dropping or rewriting the line it lets through.
|
||||
if (['bootstrap-cell', 'same-cap-cell'].includes(config.mode)) {
|
||||
expected.push([
|
||||
/^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/,
|
||||
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
|
||||
@@ -454,18 +460,22 @@ function cellPlan(plan, changes, config) {
|
||||
const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode)
|
||||
// Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly.
|
||||
const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined
|
||||
// Only a template stale enough to predate the line may move it, and only by gaining it;
|
||||
// requireDesiredStartupScript holds the after value to the exact reviewed identity.
|
||||
const stripCapacityIdentity =
|
||||
['bootstrap-cell', 'same-cap-cell'].includes(config.mode)
|
||||
if (
|
||||
typeof beforeScript !== 'string' ||
|
||||
(sameCap && relayImage(beforeScript) !== config.rollbackImage) ||
|
||||
normalizedStartupScript(
|
||||
beforeScript,
|
||||
config.mode === 'bootstrap-cell',
|
||||
stripCapacityIdentity,
|
||||
config.mode === 'same-cap-cell',
|
||||
sameCap,
|
||||
stripPool
|
||||
) !== normalizedStartupScript(
|
||||
script,
|
||||
config.mode === 'bootstrap-cell',
|
||||
stripCapacityIdentity,
|
||||
config.mode === 'same-cap-cell',
|
||||
sameCap,
|
||||
stripPool
|
||||
@@ -501,7 +511,7 @@ export function validateCapacityPlan(plan, config) {
|
||||
throw new Error('capacity Terraform plans may change only a cell')
|
||||
}
|
||||
if (
|
||||
config.mode === 'bootstrap-cell' &&
|
||||
['bootstrap-cell', 'same-cap-cell'].includes(config.mode) &&
|
||||
!SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '')
|
||||
) {
|
||||
throw new Error('capacity Terraform plan has an invalid service account')
|
||||
|
||||
@@ -428,10 +428,14 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
||||
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
||||
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
||||
const startup = ({ selectedImage, cap = 1_000, trust = false }) => [
|
||||
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
||||
...(capacity === null
|
||||
? []
|
||||
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
||||
...(trust ? [
|
||||
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
||||
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
|
||||
@@ -468,6 +472,7 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
|
||||
mode: 'same-cap-cell',
|
||||
image,
|
||||
rollbackImage,
|
||||
capacityServiceAccount: capacityIdentity,
|
||||
rehomeDirectorServiceAccount: directorIdentity,
|
||||
rehomeAudience: audience,
|
||||
regionalRehomeProtocol: '1'
|
||||
@@ -653,10 +658,12 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
||||
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
||||
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
||||
const startup = ({ selectedImage, trust = false }) => [
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
||||
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
||||
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
|
||||
...(trust ? [
|
||||
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
|
||||
@@ -693,6 +700,7 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
|
||||
mode: 'same-cap-cell',
|
||||
image,
|
||||
rollbackImage,
|
||||
capacityServiceAccount: capacityIdentity,
|
||||
rehomeDirectorServiceAccount: directorIdentity,
|
||||
rehomeAudience: audience,
|
||||
regionalRehomeProtocol: '0'
|
||||
@@ -737,6 +745,133 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
|
||||
}
|
||||
})
|
||||
|
||||
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
||||
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
||||
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
||||
const startup = ({ selectedImage, capacity }) => [
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
||||
...(capacity === null
|
||||
? []
|
||||
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
|
||||
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
|
||||
`docker pull '${selectedImage}'`,
|
||||
'docker run --detach \\',
|
||||
' --name orca-relay \\',
|
||||
` '${selectedImage}'`
|
||||
].join('\n')
|
||||
const plan = (beforeCapacity, afterCapacity) => ({
|
||||
resource_changes: [
|
||||
{
|
||||
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
|
||||
change: {
|
||||
actions: ['create', 'delete'],
|
||||
before: {
|
||||
metadata_startup_script: startup({
|
||||
selectedImage: rollbackImage,
|
||||
capacity: beforeCapacity
|
||||
})
|
||||
},
|
||||
after: {
|
||||
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
|
||||
self_link: null
|
||||
},
|
||||
after_unknown: { self_link: true }
|
||||
}
|
||||
},
|
||||
{
|
||||
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
|
||||
change: {
|
||||
actions: ['update'],
|
||||
before: { target_size: 1, version: [{ instance_template: 'old' }] },
|
||||
after: { target_size: 1, version: [{ instance_template: null }] },
|
||||
after_unknown: { version: [{ instance_template: true }] }
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
const config = {
|
||||
cellId: 'production-gce-c17',
|
||||
hardCap: 600,
|
||||
unobservedBound: 60,
|
||||
mode: 'same-cap-cell',
|
||||
image,
|
||||
rollbackImage,
|
||||
capacityServiceAccount: capacityIdentity,
|
||||
rehomeDirectorServiceAccount: directorIdentity,
|
||||
rehomeAudience: audience,
|
||||
regionalRehomeProtocol: '0'
|
||||
}
|
||||
// A template old enough to predate the line gains it, which is the only move allowed.
|
||||
assert.deepEqual(
|
||||
validateCapacityPlan(plan(null, capacityIdentity), config),
|
||||
{ mode: 'same-cap-cell', changes: 2 }
|
||||
)
|
||||
assert.deepEqual(
|
||||
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
|
||||
{ mode: 'same-cap-cell', changes: 2 }
|
||||
)
|
||||
for (const [before, after] of [
|
||||
[capacityIdentity, null],
|
||||
[null, null],
|
||||
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
|
||||
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
|
||||
]) {
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(plan(before, after), config),
|
||||
/reviewed image and capacity/,
|
||||
`${before} -> ${after}`
|
||||
)
|
||||
}
|
||||
// Without the pin there is nothing reviewing the line the comparison now ignores.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
||||
...config,
|
||||
capacityServiceAccount: undefined
|
||||
}),
|
||||
/invalid service account/
|
||||
)
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(plan(null, capacityIdentity), {
|
||||
...config,
|
||||
capacityServiceAccount: 'not-an-email'
|
||||
}),
|
||||
/invalid service account/
|
||||
)
|
||||
})
|
||||
|
||||
test('the capacity identity argument is required by same-cap-cell mode', () => {
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
const base = [
|
||||
'--mode', 'same-cap-cell',
|
||||
'--cell-id', 'production-gce-c17',
|
||||
'--hard-cap', '600',
|
||||
'--unobserved-bound', '60',
|
||||
'--image', image,
|
||||
'--rollback-image', rollbackImage,
|
||||
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
||||
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
||||
'--regional-rehome-protocol', '0'
|
||||
]
|
||||
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
|
||||
assert.throws(
|
||||
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
|
||||
/--capacity-service-account is invalid/
|
||||
)
|
||||
assert.equal(
|
||||
parseCapacityPlanArguments([
|
||||
...base,
|
||||
'--capacity-service-account',
|
||||
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
||||
]).capacityServiceAccount,
|
||||
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
||||
)
|
||||
})
|
||||
|
||||
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
@@ -747,6 +882,7 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
|
||||
'--unobserved-bound', '60',
|
||||
'--image', image,
|
||||
'--rollback-image', rollbackImage,
|
||||
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
||||
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
||||
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
||||
...extra
|
||||
@@ -788,10 +924,12 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
|
||||
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
|
||||
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
|
||||
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
|
||||
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
|
||||
const audience = 'https://relay.example.com/v1/admin/host-drain'
|
||||
const startup = ({ selectedImage, pool }) => [
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
|
||||
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
|
||||
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
|
||||
...(pool === undefined
|
||||
? []
|
||||
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
|
||||
@@ -837,6 +975,7 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
|
||||
mode: 'same-cap-cell',
|
||||
image,
|
||||
rollbackImage,
|
||||
capacityServiceAccount: capacityIdentity,
|
||||
rehomeDirectorServiceAccount: directorIdentity,
|
||||
rehomeAudience: audience,
|
||||
regionalRehomeProtocol: '1'
|
||||
@@ -891,6 +1030,7 @@ test('the database pool argument is accepted by same-cap-cell mode alone', () =>
|
||||
'--unobserved-bound', '60',
|
||||
'--image', image,
|
||||
'--rollback-image', rollbackImage,
|
||||
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
|
||||
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
|
||||
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
|
||||
'--regional-rehome-protocol', '1',
|
||||
|
||||
@@ -385,6 +385,13 @@ director; the workflow never receives or mints a director or stamped-cell runtim
|
||||
keeps only the selected cell migration-only, while the exact rollback digest remains dispatchable via
|
||||
the same workflow's `rollback` mode.
|
||||
|
||||
A roll holds the whole startup script identical before and after except the image, so a
|
||||
template stale enough to predate a pinned line fails closed rather than absorbing the drift.
|
||||
The one exception is the capacity identity: a cell that predates it gains it on its next roll,
|
||||
and the plan validator pins the exact reviewed identity instead of comparing that line, so a
|
||||
roll can never drop or rewrite it. Any other stale line still fails closed and needs a
|
||||
convergence apply first.
|
||||
|
||||
C17 and C18 hold no hosts and are not general, so rolling one displaces nobody: they are the
|
||||
zero-displacement canary for a new image. Their wave enters and leaves migration-only, so its
|
||||
isolate and its restore are both no-ops and the selector generation does not move; a general
|
||||
|
||||
Reference in New Issue
Block a user