Files
orca/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs
T
Jinwoo Hong e9bf6d1c3f fix(relay): the cell switch tool changes only the named switches (#27056)
* fix(relay): the cell switch tool changes only the named switches and checks every key on read-back

A cell resets a key missing from its switch file to its default, so the tool now reads the
current object and changes only what --set names. The read-back compares every known key and
fails when the cell reports ignoredKeys.

* fix(relay): the cell switch tool refuses a switch the cell's image does not support

It reads supportedFlags from runtime-status (#27057). An image from before that field takes only
readinessLocal and ticketCheck off/shadow, so ticketCheck=enforce cannot void an old cell's object.

* fix(relay): the switch tool validates the whole object, refuses voided files, and records admitMode in Postgres

Every key and value of the object to be written is checked against what the image accepts (an
image without the list: only the keys it applies, ticketCheck off/shadow). A file the cell did
not apply is refused. admitMode=reserve needs a typed RESERVE <cell>; Postgres records reserve
before the file is written, and db only after the cell reports every control leased again.

* feat(relay): the switch tool knows reregisterInFlight (1-16), the flip-back pace

* fix(relay): the switch tool closes the cache gap before reserve, undoes a refused write, and has a break-glass db flip

After recording reserve in Postgres it waits past every director's 5 s cache and records it
again (re-running the open-flow check) before writing the file. A file write refused outright
(412 or 4xx) puts Postgres back to db; after a 5xx it stays reserve. An object the cell did not
apply is repaired when the new object passes the image check. Break glass (typed BREAK GLASS
<cell>) records db for a dead or unreachable reserve cell without its read-back.

* fix(relay): a director deploy refuses a traffic move across reserve-mode support while any cell is reserve

* fix(relay): the flip-back wait scales with the cell's controls and pace, and says how to finish

* fix(relay): the switch tool refuses to re-arm a tripped cell, breaks glass only on a silent one, and undoes reserve if the second post is refused

* feat(relay): director deploys set reserve placement, and refuse turning it off while any cell is reserve

* fix(relay): Terraform keeps the deploy-owned placement and shadow cells, and the deploy guard reads placement as built and refuses dropping a reserve cell from the shadow list

* fix(relay): director deploys take --admin-audience alone, name a drifted deploy identity, and carry the shadow seat-feed cells

Staging passes --admin-audience for the reserve guard without durable rehome
verification, which the parser refused. Only the generation and control
origin now travel together, and they still need the audience.

The guard decodes the caller from its token and refuses, by name, a revision
whose ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT is another account, instead of
reading into a 401.

--shadow-seat-feed-cells (preserve, all, or a cell list) is a workflow input
on both director deploys; preserve keeps the serving value. A contract test
parses every workflow's real argv for both deploy scripts.

* fix(relay): the flag tool paces a flip back from the pool size the cell reports

An image from before the field falls back to the deployed sizes.
2026-10-10 19:41:47 -04:00

141 lines
5.6 KiB
JavaScript

import { spawnSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
const SECRET = 'orca-cloud-relay-regional-placement-enabled'
function validate(input) {
for (const key of ['project', 'region', 'service', 'bootstrap_version']) {
if (typeof input?.[key] !== 'string' || !input[key] || /[\r\n]/.test(input[key])) {
throw new Error(`invalid ${key}`)
}
}
if (!/^[a-z][a-z0-9-]{0,62}$/.test(input.service)) throw new Error('invalid service')
if (!/^[1-9][0-9]*$/.test(input.bootstrap_version)) {
throw new Error('invalid bootstrap_version')
}
}
export function classifyRelayServiceDescribeFailure(args, stderr) {
const serviceDescribe = args[0] === 'run' && args[1] === 'services' && args[2] === 'describe'
if (serviceDescribe && (stderr.includes('NOT_FOUND') || /Cannot find service \[[^\]\r\n]+\]/.test(stderr))) {
return 'NOT_FOUND'
}
return 'GCLOUD_FAILED'
}
function defaultRun(args) {
const result = spawnSync('gcloud', args, {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
maxBuffer: 10 * 1024 * 1024
})
if (result.status !== 0) {
const error = new Error('gcloud read failed')
error.code = classifyRelayServiceDescribeFailure(args, result.stderr)
throw error
}
return JSON.parse(result.stdout)
}
function gcloudArguments(kind, input, revision) {
return [
'run', kind, 'describe', revision ?? input.service,
'--project', input.project,
'--region', input.region,
'--format=json'
]
}
export function readRelayServingRegionalPlacementVersion(input, dependencies = {}) {
validate(input)
const run = dependencies.run ?? defaultRun
let service
try {
service = run(gcloudArguments('services', input))
} catch (error) {
if (error?.code === 'NOT_FOUND') {
return { version: input.bootstrap_version, cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }
}
throw error
}
const serving = (service.status?.traffic ?? []).filter(
(entry) => Number(entry.percent ?? 0) > 0
)
if (
serving.length !== 1 ||
Number(serving[0].percent) !== 100 ||
typeof serving[0].revisionName !== 'string'
) {
throw new Error('Relay director must have exactly one revision serving 100% traffic')
}
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter((environment) =>
environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT')
)
if (cohortSettings.length > 1 || (cohortSettings.length === 1 &&
(typeof cohortSettings[0].value !== 'string' ||
!/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) {
throw new Error('serving region correction cohort is invalid')
}
const cohort_percent = cohortSettings[0]?.value ?? '0'
// Set by the director deploy (placement) or by hand (shadow cells), never by Terraform: carried
// so an apply keeps them, and empty when the serving revision has none.
const reserve_placement = servingPlainSetting(revision, 'ORCA_RELAY_RESERVE_PLACEMENT', /^(?:off|dry-run|on)$/)
const shadow_seat_feed_cells = servingPlainSetting(
revision,
'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS',
/^(?:all|[A-Za-z0-9_-]+(?:, ?[A-Za-z0-9_-]+)*)$/
)
const carried = { cohort_percent, reserve_placement, shadow_seat_feed_cells }
const references = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter(
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
)
)
if (references.length === 0) return { version: input.bootstrap_version, ...carried }
const reference = normalizeSecretReference(references[0])
if (
references.length !== 1 ||
reference?.secret !== SECRET ||
!/^[1-9][0-9]*$/.test(reference?.version ?? '')
) {
throw new Error('serving regional placement secret reference is invalid')
}
return { version: reference.version, ...carried }
}
// One plain value, or none; anything else (a secret, a duplicate, a malformed value) fails the
// plan rather than letting an apply drop or reset the setting.
function servingPlainSetting(revision, name, valid) {
const settings = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter((environment) => environment.name === name)
)
if (settings.length === 0) return ''
if (settings.length > 1 || typeof settings[0].value !== 'string' || !valid.test(settings[0].value)) {
throw new Error(`serving ${name} is invalid`)
}
return settings[0].value
}
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
// `gcloud run revisions describe --format=json` emits the Knative v1 shape
// `valueFrom.secretKeyRef.{name,key}`, where `name` may be a full resource path.
// A `key` of "latest" is deliberately left invalid: the director's serving
// version must be a pinned integer for this data source to mean anything.
export function normalizeSecretReference(environment) {
const v2 = environment?.valueSource?.secretKeyRef
if (v2) return { secret: v2.secret, version: v2.version }
const v1 = environment?.valueFrom?.secretKeyRef
if (!v1) return undefined
const secret = typeof v1.name === 'string' ? v1.name.replace(/^projects\/[^/]+\/secrets\//, '') : undefined
return { secret, version: v1.key }
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const chunks = []
for await (const chunk of process.stdin) chunks.push(chunk)
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'))
process.stdout.write(`${JSON.stringify(readRelayServingRegionalPlacementVersion(input))}\n`)
}