mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
* chore(relay): target each cell's backend service from the same-cap job
The 60 s connection drain timeout merged in #21848 has no safe apply path.
A root plan scoped to the backend services alone still pulls every
`google_compute_instance_template.relay_gce_cell` in as a dependency, and
standing image drift turns all 29 into replacements, so applying it would roll
the fleet at once.
Add `google_compute_backend_service.relay_gce_cell["${TARGET_CELL_ID}"]` to
both plan invocations in the per-cell same-cap job, next to the template and
MIG it already targets, and teach the reviewed plan validator to allow exactly
one extra change: an in-place update of that one cell's backend whose only
changed attribute is `connection_draining_timeout_sec`, landing on the
constant `validate-relay-asia-topology-plan.mjs` exports. Any other attribute,
any other resource, or a backend for another cell still fails the validator.
The accepted update is reported as `connectionDrainUpdate` and kept out of
`changes`, so the apply step's stranded branch and the resume step's drift
branch keep reading the template-and-MIG count they were written against; the
resume branch additionally accepts a plan whose only pending change is that
drain update, which restarts nothing.
Claude-Session: relay-same-cap-targets-cell-backend
* fix(relay): also let the same-cap wave apply this cell's LB request logging
A read-only production plan for production-gce-c7 showed the live US cell
backends carry no `log_config` at all, while relay-gce-cells.tf has declared
`log_config { enable = true, sample_rate = var.relay_gce_cell_log_sample_rate }`
on every cell backend since the Terraform root landed in 3eec77c11a (#18413).
Nothing has applied it because every production apply since is a per-cell
targeted plan that names only the template and the MIG.
So the real canary plan's backend moves two paths, not one:
`["connection_draining_timeout_sec", "log_config.0"]`. The drain-only validator
rejected exactly that plan, which would have stranded the cell mid-wave after
the drain had already started.
Accept both, each optional, for this cell's backend only: the drain landing on
RELAY_CELL_CONNECTION_DRAIN_SECONDS, and a log_config of exactly one block with
`enable = true` and `sample_rate` equal to RELAY_CELL_LOG_SAMPLE_RATE, the
declared default of a variable no environment file overrides. Any third path,
a different sample rate, disabled logging, another cell, or a replacement still
fails. The accepted paths are reported as `backendUpdate`, which the resume
branch now reads instead of the drain-only flag.
Verified against the real production plan: the masked, three-target plan for
production-gce-c7 contains exactly that cell's template, MIG, and backend
service and nothing else, and this validator returns
`{"changes":2,"backendUpdate":["connection_draining_timeout_sec","log_config.0"]}`.
Claude-Session: relay-same-cap-targets-cell-backend
652 lines
26 KiB
JavaScript
652 lines
26 KiB
JavaScript
import { readFileSync } from 'node:fs'
|
|
import { pathToFileURL } from 'node:url'
|
|
import {
|
|
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
|
|
RELAY_CELL_LOG_SAMPLE_RATE
|
|
} from './validate-relay-asia-topology-plan.mjs'
|
|
|
|
const CELL_BACKEND_RESOURCE = 'google_compute_backend_service.relay_gce_cell'
|
|
const CONNECTION_DRAIN_PATH = 'connection_draining_timeout_sec'
|
|
// A backend with no logging has `log_config: []`, so gaining the block moves this one path.
|
|
const CELL_LOG_CONFIG_PATH = 'log_config.0'
|
|
|
|
const SERVICE_ACCOUNT_EMAIL =
|
|
/^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com$/
|
|
|
|
const REHOME_CONFIG =
|
|
/^ printf 'ORCA_RELAY_REHOME_(?:DIRECTOR_SERVICE_ACCOUNT|AUDIENCE)=%s\\n' '[^'\n]+'$/
|
|
|
|
const DATABASE_POOL_MAX = /^ printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '[0-9]+'$/
|
|
|
|
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
|
|
function rehomeProtocol({ regionalRehomeProtocol }) {
|
|
if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) {
|
|
throw new Error('same-cap Terraform plan has an invalid regional rehome protocol')
|
|
}
|
|
return Number(regionalRehomeProtocol)
|
|
}
|
|
|
|
// Only cells off the root pool default get a pool line, so the caller states whether to expect one.
|
|
function databasePoolMax({ databasePoolMax: value }) {
|
|
if (value === undefined) return undefined
|
|
const pool = Number(value)
|
|
if (!/^[0-9]+$/.test(String(value)) || pool < 1 || pool > 100) {
|
|
throw new Error('same-cap Terraform plan has an invalid database pool max')
|
|
}
|
|
return String(pool)
|
|
}
|
|
|
|
export function parseCapacityPlanArguments(argv) {
|
|
const values = {}
|
|
for (let index = 0; index < argv.length; index += 2) {
|
|
const key = argv[index]
|
|
const value = argv[index + 1]
|
|
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
|
values[key.slice(2)] = value
|
|
}
|
|
for (const key of ['mode', 'cell-id', 'hard-cap', 'unobserved-bound']) {
|
|
if (!values[key]) throw new Error(`missing --${key}`)
|
|
}
|
|
if (!['cell', 'bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(values.mode)) {
|
|
throw new Error('--mode must be cell, bootstrap-cell, same-cap-cell, or same-cap-image')
|
|
}
|
|
const integer = (key) => {
|
|
const value = Number(values[key])
|
|
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`--${key} is invalid`)
|
|
return value
|
|
}
|
|
if (!values.image) throw new Error('missing --image')
|
|
if (
|
|
['bootstrap-cell', 'same-cap-cell'].includes(values.mode) &&
|
|
!values['capacity-service-account']
|
|
) {
|
|
throw new Error('missing --capacity-service-account')
|
|
}
|
|
if (
|
|
values.mode === 'same-cap-cell' &&
|
|
(!values['rollback-image'] ||
|
|
!values['rehome-director-service-account'] ||
|
|
!values['rehome-audience'] ||
|
|
!['0', '1', '3'].includes(values['regional-rehome-protocol']))
|
|
) throw new Error('same-cap validation requires rollback image and rehome trust config')
|
|
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
|
|
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
|
|
}
|
|
if (values.mode !== 'same-cap-cell' && values['database-pool-max'] !== undefined) {
|
|
throw new Error('--database-pool-max applies only to same-cap-cell validation')
|
|
}
|
|
if (values.mode === 'same-cap-image' && !values['rollback-image']) {
|
|
throw new Error('same-cap image validation requires a rollback image')
|
|
}
|
|
if (
|
|
values['capacity-service-account'] !== undefined &&
|
|
!SERVICE_ACCOUNT_EMAIL.test(values['capacity-service-account'])
|
|
) {
|
|
throw new Error('--capacity-service-account is invalid')
|
|
}
|
|
return {
|
|
mode: values.mode,
|
|
cellId: values['cell-id'],
|
|
hardCap: integer('hard-cap'),
|
|
unobservedBound: integer('unobserved-bound'),
|
|
image: values.image,
|
|
capacityServiceAccount: values['capacity-service-account'],
|
|
rollbackImage: values['rollback-image'],
|
|
rehomeDirectorServiceAccount: values['rehome-director-service-account'],
|
|
rehomeAudience: values['rehome-audience'],
|
|
regionalRehomeProtocol: values['regional-rehome-protocol'],
|
|
databasePoolMax: values['database-pool-max']
|
|
}
|
|
}
|
|
|
|
function mutations(plan) {
|
|
if (!Array.isArray(plan.resource_changes)) throw new Error('Terraform plan has no resource changes')
|
|
return plan.resource_changes.filter(({ change }) => {
|
|
const actions = change?.actions
|
|
return Array.isArray(actions) && !actions.every((action) => ['no-op', 'read'].includes(action))
|
|
})
|
|
}
|
|
|
|
function sameActions(change, expected) {
|
|
return JSON.stringify(change.change?.actions) === JSON.stringify(expected)
|
|
}
|
|
|
|
function changedPaths(before, after, path = []) {
|
|
if (Object.is(before, after)) return []
|
|
const beforeObject = before !== null && typeof before === 'object'
|
|
const afterObject = after !== null && typeof after === 'object'
|
|
if (!beforeObject || !afterObject || Array.isArray(before) !== Array.isArray(after)) {
|
|
return [path.join('.')]
|
|
}
|
|
const keys = new Set([...Object.keys(before), ...Object.keys(after)])
|
|
return [...keys].flatMap((key) => changedPaths(before[key], after[key], [...path, key]))
|
|
}
|
|
|
|
function unknownPaths(value, path = []) {
|
|
if (value === true) return [path.join('.')]
|
|
if (value === null || typeof value !== 'object') return []
|
|
return Object.entries(value).flatMap(([key, nested]) => unknownPaths(nested, [...path, key]))
|
|
}
|
|
|
|
function valueAtPath(value, path) {
|
|
return path.split('.').reduce((current, key) => current?.[key], value)
|
|
}
|
|
|
|
function providerDefaultPaths(change, paths) {
|
|
const empty = (value) =>
|
|
value === '' ||
|
|
value === 0 ||
|
|
(Array.isArray(value) && value.length === 0) ||
|
|
(value !== null &&
|
|
typeof value === 'object' &&
|
|
!Array.isArray(value) &&
|
|
Object.keys(value).length === 0)
|
|
return paths.filter(
|
|
(path) =>
|
|
empty(valueAtPath(change.change.before, path)) &&
|
|
valueAtPath(change.change.after, path) === null
|
|
)
|
|
}
|
|
|
|
function canonicalResourcePaths(change, paths) {
|
|
const canonical = (value) =>
|
|
typeof value === 'string'
|
|
? value.replace('https://www.googleapis.com/compute/v1/', '')
|
|
: value
|
|
return paths.filter(
|
|
(path) =>
|
|
canonical(valueAtPath(change.change.before, path)) ===
|
|
canonical(valueAtPath(change.change.after, path))
|
|
)
|
|
}
|
|
|
|
function bootstrapRestartNormalizationPaths(change, mode) {
|
|
if (!['bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(mode)) return []
|
|
const policyMatches =
|
|
valueAtPath(change.change.before, 'update_policy.0.minimal_action') === 'RESTART' &&
|
|
valueAtPath(change.change.after, 'update_policy.0.minimal_action') === 'REPLACE'
|
|
const priorVersion = valueAtPath(change.change.before, 'version.0.name')
|
|
const versionMatches =
|
|
typeof priorVersion === 'string' &&
|
|
/^0\/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}\+00:00$/.test(priorVersion) &&
|
|
valueAtPath(change.change.after, 'version.0.name') === 'primary'
|
|
return policyMatches && versionMatches
|
|
? ['update_policy.0.minimal_action', 'version.0.name']
|
|
: []
|
|
}
|
|
|
|
function requireOnlyPaths(change, allowed, required = [], allowedUnknown = new Set()) {
|
|
const paths = changedPaths(change.change.before, change.change.after)
|
|
const unknown = unknownPaths(change.change.after_unknown)
|
|
const unexpected = paths.filter((path) => !allowed.has(path))
|
|
const unexpectedUnknown = unknown.filter((path) => !allowedUnknown.has(path))
|
|
if (
|
|
unexpected.length > 0 ||
|
|
unexpectedUnknown.length > 0 ||
|
|
required.some((path) => !paths.includes(path))
|
|
) {
|
|
throw new Error(`${change.address} changes outside the reviewed capacity fields`)
|
|
}
|
|
}
|
|
|
|
function relayImage(script) {
|
|
const lines = script.split('\n')
|
|
const starts = lines.flatMap((line, index) =>
|
|
line === 'docker run --detach \\' ? [index] : [])
|
|
const commands = starts.map((start) => {
|
|
const end = lines.findIndex((line, index) => index > start && !line.endsWith(' \\'))
|
|
return end < 0 ? [] : lines.slice(start, end + 1)
|
|
})
|
|
const relayCommands = commands.filter((command) =>
|
|
command.filter((line) => line === ' --name orca-relay \\').length === 1)
|
|
if (relayCommands.length !== 1) return null
|
|
const command = relayCommands[0]
|
|
const image = /^ '([^'\n]+@sha256:[a-f0-9]{64})'$/.exec(command.at(-1))?.[1]
|
|
const digests = [...command.join('\n').matchAll(/'([^'\n]+@sha256:[a-f0-9]{64})'/g)]
|
|
return image && digests.length === 1 ? image : null
|
|
}
|
|
|
|
function normalizedStartupScript(
|
|
script,
|
|
stripCapacityIdentity = false,
|
|
stripRehomeConfig = false,
|
|
preserveCapacity = false,
|
|
stripDatabasePoolMax = false
|
|
) {
|
|
const image = relayImage(script)
|
|
if (!image) throw new Error('cell plan startup script has no Relay image')
|
|
const digest = image.split('@')[1]
|
|
const capacityAssignment =
|
|
/^ printf 'ORCA_RELAY_CELL_CONNECTION_(?:HARD_CAP|UNOBSERVED_BOUND)=%s\\n' '[0-9]+'$/
|
|
const capacityIdentity =
|
|
/^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/
|
|
return script
|
|
.split('\n')
|
|
.filter(
|
|
(line) =>
|
|
(preserveCapacity || !capacityAssignment.test(line)) &&
|
|
(!stripCapacityIdentity || !capacityIdentity.test(line)) &&
|
|
(!stripRehomeConfig || !REHOME_CONFIG.test(line)) &&
|
|
(!stripDatabasePoolMax || !DATABASE_POOL_MAX.test(line))
|
|
)
|
|
.join('\n')
|
|
.replaceAll(image, '<relay-image>')
|
|
.replaceAll(digest, '<relay-image-digest>')
|
|
}
|
|
|
|
function hasExactSingleAssignment(lines, pattern, expected) {
|
|
const assignments = lines.filter((line) => pattern.test(line))
|
|
return assignments.length === 1 && assignments[0] === expected
|
|
}
|
|
|
|
function requireDesiredStartupScript(script, config) {
|
|
const lines = typeof script === 'string' ? script.split('\n') : []
|
|
const expected = [
|
|
[
|
|
/^ printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '[0-9]+'$/,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${config.hardCap}'`
|
|
],
|
|
[
|
|
/^ printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '[0-9]+'$/,
|
|
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'`
|
|
]
|
|
]
|
|
// A same-cap cell whose template predates this line gains it on its next roll, so the
|
|
// before/after comparison ignores it; pinning the exact identity here is what reviews it,
|
|
// and what stops a roll dropping or rewriting the line it lets through.
|
|
if (['bootstrap-cell', 'same-cap-cell'].includes(config.mode)) {
|
|
expected.push([
|
|
/^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/,
|
|
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
|
|
])
|
|
}
|
|
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) >= 1
|
|
if (rehomeTrusted) {
|
|
expected.push(
|
|
[
|
|
/^ printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '[^'\n]+'$/,
|
|
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${config.rehomeDirectorServiceAccount}'`
|
|
],
|
|
[
|
|
/^ printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '[^'\n]+'$/,
|
|
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${config.rehomeAudience}'`
|
|
]
|
|
)
|
|
}
|
|
// A protocol-0 cell is not a rehome source, so gaining any rehome trust line is real drift.
|
|
const unexpectedRehome =
|
|
config.mode === 'same-cap-cell' &&
|
|
!rehomeTrusted &&
|
|
lines.some((line) => REHOME_CONFIG.test(line))
|
|
const pool = config.mode === 'same-cap-cell' ? databasePoolMax(config) : undefined
|
|
if (pool !== undefined) {
|
|
expected.push([
|
|
DATABASE_POOL_MAX,
|
|
` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`
|
|
])
|
|
}
|
|
// An unpinned cell sits on the root pool default, so gaining a pool line is real drift.
|
|
const unexpectedDatabasePoolMax =
|
|
config.mode === 'same-cap-cell' &&
|
|
pool === undefined &&
|
|
lines.some((line) => DATABASE_POOL_MAX.test(line))
|
|
if (
|
|
typeof script !== 'string' ||
|
|
relayImage(script) !== config.image ||
|
|
unexpectedRehome ||
|
|
unexpectedDatabasePoolMax ||
|
|
expected.some(([pattern, line]) => !hasExactSingleAssignment(lines, pattern, line))
|
|
) {
|
|
throw new Error('cell plan does not contain the reviewed image and capacity')
|
|
}
|
|
}
|
|
|
|
// The same-cap job targets this cell's backend service so the two declared-but-unapplied
|
|
// settings land one cell at a time: an unindexed root plan pulls the whole MIG and template
|
|
// resources in as dependencies, which standing image drift turns into a 29-cell roll. Each
|
|
// attribute is optional because a cell that already has it plans no change for it.
|
|
// Splitting the backend out here keeps `changes` the template-and-MIG count both callers read.
|
|
function takeCellBackendUpdate(changes, config) {
|
|
const backends = changes.filter(
|
|
({ address }) => typeof address === 'string' && address.startsWith(`${CELL_BACKEND_RESOURCE}[`)
|
|
)
|
|
if (config.mode !== 'same-cap-cell' || backends.length === 0) {
|
|
return { rest: changes, backendUpdate: [] }
|
|
}
|
|
const [backend] = backends
|
|
if (
|
|
backends.length !== 1 ||
|
|
backend.address !== `${CELL_BACKEND_RESOURCE}[${JSON.stringify(config.cellId)}]` ||
|
|
backend.deposed !== undefined ||
|
|
!sameActions(backend, ['update'])
|
|
) {
|
|
throw new Error('cell plan may change only this cell backend drain and request logging')
|
|
}
|
|
const after = backend.change?.after
|
|
const moved = changedPaths(backend.change?.before, after)
|
|
const logging = after?.log_config?.[0]
|
|
const accepted = [CONNECTION_DRAIN_PATH, CELL_LOG_CONFIG_PATH].filter((path) =>
|
|
moved.includes(path))
|
|
if (
|
|
accepted.length === 0 ||
|
|
(moved.includes(CONNECTION_DRAIN_PATH) &&
|
|
after?.[CONNECTION_DRAIN_PATH] !== RELAY_CELL_CONNECTION_DRAIN_SECONDS) ||
|
|
(moved.includes(CELL_LOG_CONFIG_PATH) &&
|
|
(after?.log_config?.length !== 1 ||
|
|
logging?.enable !== true ||
|
|
logging?.sample_rate !== RELAY_CELL_LOG_SAMPLE_RATE))
|
|
) {
|
|
throw new Error('cell plan may change only this cell backend drain and request logging')
|
|
}
|
|
const backendComputed = new Set(['fingerprint', 'generated_id'])
|
|
requireOnlyPaths(
|
|
backend,
|
|
new Set([
|
|
...accepted,
|
|
...unknownPaths(backend.change.after_unknown).filter((path) => backendComputed.has(path))
|
|
]),
|
|
accepted,
|
|
backendComputed
|
|
)
|
|
return { rest: changes.filter((change) => change !== backend), backendUpdate: accepted }
|
|
}
|
|
|
|
function plannedResources(module) {
|
|
if (!module) return []
|
|
return [
|
|
...(module.resources ?? []),
|
|
...(module.child_modules ?? []).flatMap(plannedResources)
|
|
]
|
|
}
|
|
|
|
function canonicalResource(value) {
|
|
return typeof value === 'string'
|
|
? value.replace('https://www.googleapis.com/compute/v1/', '')
|
|
: value
|
|
}
|
|
|
|
function requireDesiredPlannedCell(plan, config) {
|
|
const resources = plannedResources(plan.planned_values?.root_module)
|
|
const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]`
|
|
const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]`
|
|
const template = resources.find(({ address }) => address === templateAddress)?.values
|
|
const manager = resources.find(({ address }) => address === managerAddress)?.values
|
|
if (!template || !manager) throw new Error('convergence plan has no exact planned C26 state')
|
|
requireDesiredStartupScript(template.metadata_startup_script, config)
|
|
const templateReference = canonicalResource(template.self_link ?? template.id)
|
|
const managerReference = canonicalResource(manager.version?.[0]?.instance_template)
|
|
if (!templateReference || templateReference !== managerReference) {
|
|
throw new Error('convergence plan does not bind the MIG to the reviewed template')
|
|
}
|
|
}
|
|
|
|
function validateManagerUpdate(manager, config) {
|
|
if (!sameActions(manager, ['update'])) {
|
|
throw new Error('cell plan has unexpected MIG actions')
|
|
}
|
|
const managerComputed = new Set([
|
|
'fingerprint',
|
|
'operation',
|
|
'status',
|
|
'version.0.instance_template'
|
|
])
|
|
const managerUnknown = unknownPaths(manager.change.after_unknown)
|
|
requireOnlyPaths(
|
|
manager,
|
|
new Set([
|
|
'version.0.instance_template',
|
|
...bootstrapRestartNormalizationPaths(manager, config.mode),
|
|
...managerUnknown.filter((path) => managerComputed.has(path))
|
|
]),
|
|
['version.0.instance_template'],
|
|
managerComputed
|
|
)
|
|
}
|
|
|
|
function requireReplacementTemplateDependency(plan, template, manager) {
|
|
const configuredManager = plan.configuration?.root_module?.resources?.find(
|
|
({ address }) => address === 'google_compute_instance_group_manager.relay_gce_cell'
|
|
)
|
|
const expectedVersionExpression = [{
|
|
instance_template: {
|
|
references: ['google_compute_instance_template.relay_gce_cell', 'each.key']
|
|
},
|
|
name: { constant_value: 'primary' }
|
|
}]
|
|
if (
|
|
JSON.stringify(configuredManager?.expressions?.version) !==
|
|
JSON.stringify(expectedVersionExpression) ||
|
|
template.change.after?.self_link != null ||
|
|
template.change.after_unknown?.self_link !== true ||
|
|
manager.change.after?.version?.[0]?.instance_template != null ||
|
|
manager.change.after_unknown?.version?.[0]?.instance_template !== true
|
|
) {
|
|
throw new Error('cell plan does not bind the MIG to the reviewed template dependency')
|
|
}
|
|
}
|
|
|
|
function cellPlan(plan, changes, config) {
|
|
const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]`
|
|
const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]`
|
|
const template = changes.find(
|
|
({ address, deposed }) => address === templateAddress && deposed === undefined
|
|
)
|
|
const manager = changes.find(({ address }) => address === managerAddress)
|
|
const obsoleteTemplates = changes.filter(
|
|
({ address, deposed }) => address === templateAddress && typeof deposed === 'string'
|
|
)
|
|
const allowsObsoleteTemplates =
|
|
config.mode === 'same-cap-image' &&
|
|
obsoleteTemplates.length > 0 &&
|
|
obsoleteTemplates.every((change) => sameActions(change, ['delete']))
|
|
if (
|
|
!template ||
|
|
!manager ||
|
|
changes.length !== 2 + obsoleteTemplates.length ||
|
|
(obsoleteTemplates.length > 0 && !allowsObsoleteTemplates)
|
|
) {
|
|
throw new Error('cell plan must change only the exact instance template and MIG')
|
|
}
|
|
if (!sameActions(template, ['create', 'delete']) || !sameActions(manager, ['update'])) {
|
|
throw new Error('cell plan has unexpected replacement actions')
|
|
}
|
|
const templateComputed = new Set([
|
|
'confidential_instance_config',
|
|
'creation_timestamp',
|
|
'disk.0.architecture',
|
|
'disk.0.interface',
|
|
'disk.0.mode',
|
|
'disk.0.provisioned_iops',
|
|
'disk.0.provisioned_throughput',
|
|
'disk.0.type',
|
|
'id',
|
|
'metadata_fingerprint',
|
|
'name',
|
|
'network_interface.0.internal_ipv6_prefix_length',
|
|
'network_interface.0.ipv6_access_type',
|
|
'network_interface.0.ipv6_address',
|
|
'network_interface.0.name',
|
|
'network_interface.0.network',
|
|
'network_interface.0.stack_type',
|
|
'network_interface.0.subnetwork_project',
|
|
'numeric_id',
|
|
'region',
|
|
'self_link',
|
|
'self_link_unique',
|
|
'tags_fingerprint'
|
|
])
|
|
const templateDefaults = [
|
|
'description',
|
|
'disk.0.disk_name',
|
|
'disk.0.guest_os_features',
|
|
'disk.0.labels',
|
|
'disk.0.resource_manager_tags',
|
|
'disk.0.resource_policies',
|
|
'disk.0.source',
|
|
'disk.0.source_snapshot',
|
|
'instance_description',
|
|
'key_revocation_action_type',
|
|
'min_cpu_platform',
|
|
'network_interface.0.network_ip',
|
|
'network_interface.0.nic_type',
|
|
'network_interface.0.queue_count',
|
|
'scheduling.0.availability_domain',
|
|
'scheduling.0.instance_termination_action',
|
|
'scheduling.0.min_node_cpus',
|
|
'scheduling.0.termination_time'
|
|
]
|
|
const templateCanonicalResources = [
|
|
'disk.0.source_image',
|
|
'network_interface.0.subnetwork'
|
|
]
|
|
const templateUnknown = unknownPaths(template.change.after_unknown)
|
|
requireOnlyPaths(
|
|
template,
|
|
new Set([
|
|
'metadata_startup_script',
|
|
...templateUnknown.filter((path) => templateComputed.has(path)),
|
|
...providerDefaultPaths(template, templateDefaults),
|
|
...canonicalResourcePaths(template, templateCanonicalResources)
|
|
]),
|
|
['metadata_startup_script'],
|
|
templateComputed
|
|
)
|
|
validateManagerUpdate(manager, config)
|
|
requireReplacementTemplateDependency(plan, template, manager)
|
|
const beforeScript = template.change.before?.metadata_startup_script
|
|
const script = template.change.after?.metadata_startup_script
|
|
requireDesiredStartupScript(script, config)
|
|
const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode)
|
|
// Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly.
|
|
const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined
|
|
// Only a template stale enough to predate the line may move it, and only by gaining it;
|
|
// requireDesiredStartupScript holds the after value to the exact reviewed identity.
|
|
const stripCapacityIdentity =
|
|
['bootstrap-cell', 'same-cap-cell'].includes(config.mode)
|
|
if (
|
|
typeof beforeScript !== 'string' ||
|
|
(sameCap && relayImage(beforeScript) !== config.rollbackImage) ||
|
|
normalizedStartupScript(
|
|
beforeScript,
|
|
stripCapacityIdentity,
|
|
config.mode === 'same-cap-cell',
|
|
sameCap,
|
|
stripPool
|
|
) !== normalizedStartupScript(
|
|
script,
|
|
stripCapacityIdentity,
|
|
config.mode === 'same-cap-cell',
|
|
sameCap,
|
|
stripPool
|
|
)
|
|
) {
|
|
throw new Error('cell plan does not contain the reviewed image and capacity')
|
|
}
|
|
}
|
|
|
|
function convergenceCellPlan(plan, changes, config) {
|
|
const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]`
|
|
const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]`
|
|
const manager = changes.find(({ address }) => address === managerAddress)
|
|
const obsoleteTemplates = changes.filter(
|
|
({ address, deposed }) => address === templateAddress && typeof deposed === 'string'
|
|
)
|
|
if (
|
|
(!manager && obsoleteTemplates.length === 0) ||
|
|
(obsoleteTemplates.length > 1 && config.mode !== 'same-cap-image') ||
|
|
changes.length !== (manager ? 1 : 0) + obsoleteTemplates.length
|
|
) {
|
|
throw new Error('cell convergence plan changes outside the exact template and MIG')
|
|
}
|
|
if (manager) validateManagerUpdate(manager, config)
|
|
if (obsoleteTemplates.some((change) => !sameActions(change, ['delete']))) {
|
|
throw new Error('cell convergence plan has unexpected obsolete-template actions')
|
|
}
|
|
requireDesiredPlannedCell(plan, config)
|
|
}
|
|
|
|
export function validateCapacityPlan(plan, config) {
|
|
if (!['cell', 'bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(config.mode)) {
|
|
throw new Error('capacity Terraform plans may change only a cell')
|
|
}
|
|
if (
|
|
['bootstrap-cell', 'same-cap-cell'].includes(config.mode) &&
|
|
!SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '')
|
|
) {
|
|
throw new Error('capacity Terraform plan has an invalid service account')
|
|
}
|
|
if (config.mode === 'same-cap-cell') {
|
|
rehomeProtocol(config)
|
|
databasePoolMax(config)
|
|
}
|
|
if (
|
|
config.mode === 'same-cap-cell' &&
|
|
(!SERVICE_ACCOUNT_EMAIL.test(config.rehomeDirectorServiceAccount ?? '') ||
|
|
!/^https:\/\/[^/]+\/v1\/admin\/host-drain$/.test(config.rehomeAudience ?? '') ||
|
|
!/^.+@sha256:[a-f0-9]{64}$/.test(config.rollbackImage ?? ''))
|
|
) throw new Error('same-cap Terraform plan has invalid rehome trust config')
|
|
if (
|
|
config.mode === 'same-cap-image' &&
|
|
!/^.+@sha256:[a-f0-9]{64}$/.test(config.rollbackImage ?? '')
|
|
) throw new Error('same-cap image Terraform plan has an invalid rollback image')
|
|
const { rest: changes, backendUpdate } = takeCellBackendUpdate(mutations(plan), config)
|
|
const backend = backendUpdate.length > 0 ? { backendUpdate } : {}
|
|
if (changes.length === 0) {
|
|
return {
|
|
mode: config.mode,
|
|
changes: 0,
|
|
...backend,
|
|
...(config.mode === 'same-cap-image' ? { changeKind: 'none' } : {})
|
|
}
|
|
}
|
|
const replacement = changes.some(
|
|
({ address, deposed, change }) =>
|
|
address === `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` &&
|
|
deposed === undefined &&
|
|
JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete'])
|
|
)
|
|
if (replacement) cellPlan(plan, changes, config)
|
|
else convergenceCellPlan(plan, changes, config)
|
|
const obsoleteTemplateOnly = changes.every(
|
|
({ address, deposed, change }) =>
|
|
address === `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` &&
|
|
typeof deposed === 'string' &&
|
|
JSON.stringify(change?.actions) === JSON.stringify(['delete'])
|
|
)
|
|
return {
|
|
mode: config.mode,
|
|
changes: changes.length,
|
|
...backend,
|
|
...(config.mode === 'same-cap-image'
|
|
? {
|
|
changeKind: replacement
|
|
? changes.some(
|
|
({ address, deposed }) =>
|
|
address === `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` &&
|
|
typeof deposed === 'string'
|
|
)
|
|
? 'replacement-with-obsolete-template'
|
|
: 'replacement'
|
|
: obsoleteTemplateOnly
|
|
? 'obsolete-template-delete'
|
|
: 'manager-convergence'
|
|
}
|
|
: {})
|
|
}
|
|
}
|
|
|
|
export function main(argv = process.argv.slice(2)) {
|
|
const config = parseCapacityPlanArguments(argv)
|
|
const plan = JSON.parse(readFileSync(0, 'utf8'))
|
|
process.stdout.write(`${JSON.stringify({ event: 'relay_capacity_plan_verified', ...validateCapacityPlan(plan, config) })}\n`)
|
|
}
|
|
|
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
try {
|
|
main()
|
|
} catch (error) {
|
|
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
|
process.exitCode = 1
|
|
}
|
|
}
|