mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
Jinwoo Hong
e9bf6d1c3f
* fix(relay): the cell switch tool changes only the named switches and checks every key on read-back A cell resets a key missing from its switch file to its default, so the tool now reads the current object and changes only what --set names. The read-back compares every known key and fails when the cell reports ignoredKeys. * fix(relay): the cell switch tool refuses a switch the cell's image does not support It reads supportedFlags from runtime-status (#27057). An image from before that field takes only readinessLocal and ticketCheck off/shadow, so ticketCheck=enforce cannot void an old cell's object. * fix(relay): the switch tool validates the whole object, refuses voided files, and records admitMode in Postgres Every key and value of the object to be written is checked against what the image accepts (an image without the list: only the keys it applies, ticketCheck off/shadow). A file the cell did not apply is refused. admitMode=reserve needs a typed RESERVE <cell>; Postgres records reserve before the file is written, and db only after the cell reports every control leased again. * feat(relay): the switch tool knows reregisterInFlight (1-16), the flip-back pace * fix(relay): the switch tool closes the cache gap before reserve, undoes a refused write, and has a break-glass db flip After recording reserve in Postgres it waits past every director's 5 s cache and records it again (re-running the open-flow check) before writing the file. A file write refused outright (412 or 4xx) puts Postgres back to db; after a 5xx it stays reserve. An object the cell did not apply is repaired when the new object passes the image check. Break glass (typed BREAK GLASS <cell>) records db for a dead or unreachable reserve cell without its read-back. * fix(relay): a director deploy refuses a traffic move across reserve-mode support while any cell is reserve * fix(relay): the flip-back wait scales with the cell's controls and pace, and says how to finish * fix(relay): the switch tool refuses to re-arm a tripped cell, breaks glass only on a silent one, and undoes reserve if the second post is refused * feat(relay): director deploys set reserve placement, and refuse turning it off while any cell is reserve * fix(relay): Terraform keeps the deploy-owned placement and shadow cells, and the deploy guard reads placement as built and refuses dropping a reserve cell from the shadow list * fix(relay): director deploys take --admin-audience alone, name a drifted deploy identity, and carry the shadow seat-feed cells Staging passes --admin-audience for the reserve guard without durable rehome verification, which the parser refused. Only the generation and control origin now travel together, and they still need the audience. The guard decodes the caller from its token and refuses, by name, a revision whose ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT is another account, instead of reading into a 401. --shadow-seat-feed-cells (preserve, all, or a cell list) is a workflow input on both director deploys; preserve keeps the serving value. A contract test parses every workflow's real argv for both deploy scripts. * fix(relay): the flag tool paces a flip back from the pool size the cell reports An image from before the field falls back to the deployed sizes.
1362 lines
52 KiB
JavaScript
1362 lines
52 KiB
JavaScript
import { spawnSync } from 'node:child_process'
|
|
import { createHash } from 'node:crypto'
|
|
import { pathToFileURL } from 'node:url'
|
|
import { isDeepStrictEqual } from 'node:util'
|
|
|
|
const POLL_INTERVAL_MS = 5_000
|
|
const MIGRATION_TIMEOUT_MS = 14 * 60 * 1000
|
|
const CONNECTION_CAPACITY_PROTOCOL = 2
|
|
export const DIRECTOR_REGIONAL_PLACEMENT_SECRET =
|
|
'orca-cloud-relay-regional-placement-enabled'
|
|
export const DIRECTOR_REGIONAL_PLACEMENT_ENV =
|
|
'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
|
|
export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
|
|
// Step 5 placement on the directors; only this deploy changes it, so the reserve guard sees it.
|
|
export const DIRECTOR_RESERVE_PLACEMENT_ENV = 'ORCA_RELAY_RESERVE_PLACEMENT'
|
|
const RESERVE_PLACEMENT_MODES = ['off', 'dry-run', 'on']
|
|
// The cells whose feeds a director polls; a reserve cell outside it has no placing director.
|
|
export const DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV = 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS'
|
|
export const DIRECTOR_REHOME_IDENTITY_ENV =
|
|
'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT'
|
|
export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE'
|
|
export const SELECTOR_ROLLBACK_TAG = 'selector-rollback'
|
|
export const SELECTOR_REVISION_MARKER = '3'
|
|
export const DIRECTOR_ADMISSION_ENVIRONMENT = Object.freeze({
|
|
ORCA_RELAY_DATABASE_POOL_MAX: '3',
|
|
ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: '2',
|
|
ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX: '128',
|
|
ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS: '5',
|
|
ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS: '4000'
|
|
})
|
|
const DIRECTOR_STARTUP_PROBE =
|
|
'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1'
|
|
|
|
export function taggedRevisionOrigin(serviceOrigin, tag) {
|
|
const url = new URL(serviceOrigin)
|
|
if (
|
|
url.protocol !== 'https:' ||
|
|
url.pathname !== '/' ||
|
|
url.search ||
|
|
url.hash ||
|
|
!url.hostname.endsWith('.run.app')
|
|
) {
|
|
throw new Error('Cloud Run service origin is not canonical')
|
|
}
|
|
if (!/^[a-z][a-z0-9-]{0,62}$/.test(tag)) throw new Error('invalid Cloud Run tag')
|
|
return `https://${tag}---${url.hostname}`
|
|
}
|
|
|
|
export function activeRevision(service) {
|
|
const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0)
|
|
if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) {
|
|
throw new Error('relay service must have exactly one revision receiving 100% traffic')
|
|
}
|
|
return active[0].revisionName
|
|
}
|
|
|
|
export function trafficTags(service) {
|
|
return (service.status?.traffic ?? [])
|
|
.map((entry) => entry.tag)
|
|
.filter((tag) => typeof tag === 'string')
|
|
}
|
|
|
|
export function taggedTraffic(service, tag) {
|
|
const traffic = (service.status?.traffic ?? []).find((entry) => entry.tag === tag)
|
|
if (!traffic?.url || !traffic.revisionName) throw new Error(`Cloud Run tag ${tag} is not ready`)
|
|
return { origin: traffic.url, revision: traffic.revisionName }
|
|
}
|
|
|
|
export function revisionEnvironment(revision) {
|
|
const entries = revision.spec?.containers?.[0]?.env ?? []
|
|
return Object.fromEntries(
|
|
entries
|
|
.filter((entry) => entry.name && 'value' in entry)
|
|
.map((entry) => [entry.name, entry.value])
|
|
)
|
|
}
|
|
|
|
export function revisionSecretEnvironment(revision) {
|
|
const entries = revision.spec?.containers?.[0]?.env ?? []
|
|
return Object.fromEntries(
|
|
entries
|
|
.map((entry) => {
|
|
const reference = entry.valueSource?.secretKeyRef ?? entry.valueFrom?.secretKeyRef
|
|
return [entry.name, reference && {
|
|
secret: reference.secret ?? reference.name,
|
|
version: reference.version ?? reference.key
|
|
}]
|
|
})
|
|
.filter(([name, reference]) => name && reference)
|
|
)
|
|
}
|
|
|
|
export function revisionMinimumInstances(revision) {
|
|
return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0)
|
|
}
|
|
|
|
export function revisionMaximumInstances(revision) {
|
|
const value = Number(revision.metadata?.annotations?.['autoscaling.knative.dev/maxScale'])
|
|
if (!Number.isSafeInteger(value) || value < 1) {
|
|
throw new Error('serving revision has no bounded maximum instance count')
|
|
}
|
|
return value
|
|
}
|
|
|
|
function hasExpectedEnvironment(environment, expected) {
|
|
return Object.entries(expected).every(([key, value]) => environment[key] === value)
|
|
}
|
|
|
|
function projectServiceAccount(config, argument) {
|
|
const value = config[argument]
|
|
if (value === undefined) return undefined
|
|
const suffix = `@${config.project}.iam.gserviceaccount.com`
|
|
const account = value.endsWith(suffix) ? value.slice(0, -suffix.length) : ''
|
|
if (!/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/.test(account)) {
|
|
throw new Error(`--${argument} must belong to the selected project`)
|
|
}
|
|
return value
|
|
}
|
|
|
|
function directorCellsJson(value, { allowMissingRegion = false } = {}) {
|
|
if (value === undefined) return undefined
|
|
if (value.length > 100_000 || /[\r\n]/.test(value)) {
|
|
throw new Error('--director-cells-json is invalid')
|
|
}
|
|
const cells = JSON.parse(value)
|
|
if (!Array.isArray(cells) || cells.length < 1 || cells.length > 100) {
|
|
throw new Error('--director-cells-json must contain 1..100 cells')
|
|
}
|
|
const ids = new Set()
|
|
for (const cell of cells) {
|
|
const keys = Object.keys(cell ?? {}).sort()
|
|
const expectedKeys = [
|
|
'capacityRequests',
|
|
'id',
|
|
'initiallyEnabled',
|
|
...(allowMissingRegion && cell?.region === undefined ? [] : ['region']),
|
|
'url',
|
|
...(cell?.connectionHardCap === undefined
|
|
? []
|
|
: ['connectionHardCap', 'connectionUnobservedBound'])
|
|
].sort()
|
|
let origin
|
|
try {
|
|
origin = new URL(cell?.url)
|
|
} catch {
|
|
throw new Error('--director-cells-json contains an invalid cell URL')
|
|
}
|
|
const cap = cell?.connectionHardCap
|
|
const bound = cell?.connectionUnobservedBound
|
|
if (
|
|
JSON.stringify(keys) !== JSON.stringify(expectedKeys) ||
|
|
!/^[a-z][a-z0-9-]{0,39}$/.test(cell.id ?? '') ||
|
|
ids.has(cell.id) ||
|
|
origin.protocol !== 'https:' ||
|
|
origin.origin !== cell.url ||
|
|
!Number.isSafeInteger(cell.capacityRequests) ||
|
|
cell.capacityRequests < 1 ||
|
|
!['us-central1', 'asia-east2'].includes(
|
|
cell.region ?? (allowMissingRegion ? 'us-central1' : undefined)
|
|
) ||
|
|
typeof cell.initiallyEnabled !== 'boolean' ||
|
|
(cap !== undefined &&
|
|
(![600, 1_000, 3_000].includes(cap) ||
|
|
!Number.isSafeInteger(bound) ||
|
|
bound < 0 ||
|
|
bound >= cap - 100))
|
|
) {
|
|
throw new Error('--director-cells-json contains an invalid cell')
|
|
}
|
|
ids.add(cell.id)
|
|
}
|
|
return JSON.stringify(
|
|
cells.map((cell) => ({
|
|
id: cell.id,
|
|
url: cell.url,
|
|
capacityRequests: cell.capacityRequests,
|
|
region: cell.region ?? 'us-central1',
|
|
initiallyEnabled: cell.initiallyEnabled,
|
|
...(cell.connectionHardCap === undefined
|
|
? {}
|
|
: {
|
|
connectionHardCap: cell.connectionHardCap,
|
|
connectionUnobservedBound: cell.connectionUnobservedBound
|
|
})
|
|
}))
|
|
)
|
|
}
|
|
|
|
export function directorTopologyChange(currentValue, desiredValue, cellId) {
|
|
const current = JSON.parse(directorCellsJson(currentValue, { allowMissingRegion: true }))
|
|
const desired = JSON.parse(directorCellsJson(desiredValue))
|
|
if (
|
|
current.length !== desired.length ||
|
|
desired.some(({ id }, index) => id !== current[index]?.id)
|
|
) {
|
|
throw new Error('director topology changes the Relay cell set or order')
|
|
}
|
|
let changed = false
|
|
for (let index = 0; index < desired.length; index += 1) {
|
|
const before = structuredClone(current[index])
|
|
const after = structuredClone(desired[index])
|
|
if (after.id === cellId) {
|
|
delete before.connectionHardCap
|
|
delete before.connectionUnobservedBound
|
|
delete after.connectionHardCap
|
|
delete after.connectionUnobservedBound
|
|
changed = !isDeepStrictEqual(current[index], desired[index])
|
|
}
|
|
if (!isDeepStrictEqual(before, after)) {
|
|
throw new Error('director topology changes fields outside the reviewed capacity pair')
|
|
}
|
|
}
|
|
if (!desired.some(({ id }) => id === cellId)) {
|
|
throw new Error('director topology omits the reviewed capacity cell')
|
|
}
|
|
return { changed, value: JSON.stringify(desired) }
|
|
}
|
|
|
|
export function directorCellSetAddition(currentValue, desiredValue) {
|
|
const current = JSON.parse(directorCellsJson(currentValue, { allowMissingRegion: true }))
|
|
const desired = JSON.parse(directorCellsJson(desiredValue))
|
|
if (desired.length < current.length) {
|
|
throw new Error('director topology addition cannot remove cells')
|
|
}
|
|
const desiredById = new Map(desired.map((cell) => [cell.id, cell]))
|
|
if (current.some((cell) => !isDeepStrictEqual(desiredById.get(cell.id), cell))) {
|
|
throw new Error('director topology addition changes an existing cell')
|
|
}
|
|
const currentIds = new Set(current.map((cell) => cell.id))
|
|
const additions = desired.filter((cell) => !currentIds.has(cell.id))
|
|
if (additions.some((cell) => cell.initiallyEnabled !== false)) {
|
|
throw new Error('director topology additions must start disabled')
|
|
}
|
|
return { changed: additions.length > 0, value: JSON.stringify(desired) }
|
|
}
|
|
|
|
export function correctionCohortPercent(value) {
|
|
if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) {
|
|
throw new Error('region correction cohort must be an integer from 0 to 100')
|
|
}
|
|
return String(value)
|
|
}
|
|
|
|
export function directorDeploymentEnvironment(config) {
|
|
const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1]
|
|
if (config.image !== undefined && imageDigest === undefined) {
|
|
throw new Error('--image must use an immutable digest for director deployments')
|
|
}
|
|
const environment = {
|
|
...DIRECTOR_ADMISSION_ENVIRONMENT,
|
|
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER,
|
|
...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest })
|
|
}
|
|
if (config['region-correction-cohort-percent'] !== undefined &&
|
|
config['region-correction-cohort-percent'] !== 'preserve') {
|
|
environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent'])
|
|
}
|
|
if (config['reserve-placement'] !== undefined && config['reserve-placement'] !== 'preserve') {
|
|
environment[DIRECTOR_RESERVE_PLACEMENT_ENV] = config['reserve-placement']
|
|
}
|
|
if (config['shadow-seat-feed-cells'] !== undefined && config['shadow-seat-feed-cells'] !== 'preserve') {
|
|
environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] = shadowSeatFeedCellsSetting(config['shadow-seat-feed-cells'])
|
|
}
|
|
const serviceAccount = projectServiceAccount(config, 'capacity-service-account')
|
|
const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account')
|
|
const rehomeDirectorServiceAccount = projectServiceAccount(
|
|
config,
|
|
'rehome-director-service-account'
|
|
)
|
|
const cellsJson = directorCellsJson(config['director-cells-json'])
|
|
if (serviceAccount !== undefined) {
|
|
environment.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT = serviceAccount
|
|
}
|
|
if (asiaProofServiceAccount !== undefined) {
|
|
environment.ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT = asiaProofServiceAccount
|
|
}
|
|
if (rehomeDirectorServiceAccount !== undefined) {
|
|
environment[DIRECTOR_REHOME_IDENTITY_ENV] = rehomeDirectorServiceAccount
|
|
environment[DIRECTOR_REHOME_AUDIENCE_ENV] = config['rehome-audience']
|
|
}
|
|
if (cellsJson !== undefined) environment.ORCA_RELAY_CELLS_JSON = cellsJson
|
|
return environment
|
|
}
|
|
|
|
// `all` or a comma list of cell ids, as the director's config reads it.
|
|
function shadowSeatFeedCellsSetting(value) {
|
|
if (value === 'all') return value
|
|
const cells = value.split(',').map((cell) => cell.trim())
|
|
if (cells.length > 100 || cells.some((cell) => !/^[a-z][a-z0-9-]{0,39}$/.test(cell))) {
|
|
throw new Error('--shadow-seat-feed-cells must be preserve, all, or a comma list of cell ids')
|
|
}
|
|
return cells.join(',')
|
|
}
|
|
|
|
export function environmentUpdateValue(environment) {
|
|
const entries = Object.entries(environment)
|
|
if (entries.every(([key, value]) => !key.includes(',') && !value.includes(','))) {
|
|
return entries.map(([key, value]) => `${key}=${value}`).join(',')
|
|
}
|
|
const delimiter = ['~', '|', '@', '%', ';'].find((candidate) =>
|
|
entries.every(([key, value]) => !key.includes(candidate) && !value.includes(candidate))
|
|
)
|
|
if (!delimiter) throw new Error('candidate environment has no safe gcloud delimiter')
|
|
return `^${delimiter}^${entries.map(([key, value]) => `${key}=${value}`).join(delimiter)}`
|
|
}
|
|
|
|
export function parseArguments(argv) {
|
|
const values = {}
|
|
for (let index = 0; index < argv.length; index += 2) {
|
|
const key = argv[index]
|
|
const value = argv[index + 1]
|
|
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
|
|
values[key.slice(2)] = value
|
|
}
|
|
const required = ['project', 'region', 'service', 'image', 'role', 'release-id']
|
|
for (const key of required) if (!values[key]) throw new Error(`missing --${key}`)
|
|
if (!['director', 'cell'].includes(values.role)) throw new Error('--role must be director or cell')
|
|
if (values['min-instances'] !== undefined && !/^(0|[1-9][0-9]*)$/.test(values['min-instances'])) {
|
|
throw new Error('--min-instances must be a nonnegative integer')
|
|
}
|
|
if (values['max-instances'] !== undefined && !/^[1-9][0-9]*$/.test(values['max-instances'])) {
|
|
throw new Error('--max-instances must be a positive integer')
|
|
}
|
|
if (values.role === 'cell') {
|
|
for (const key of ['director-origin', 'admin-audience']) {
|
|
if (!values[key]) throw new Error(`missing --${key}`)
|
|
}
|
|
if (!['enabled', 'disabled'].includes(values['final-admission'] ?? 'enabled')) {
|
|
throw new Error('--final-admission must be enabled or disabled')
|
|
}
|
|
if (
|
|
values['capacity-service-account'] !== undefined ||
|
|
values['director-cells-json'] !== undefined ||
|
|
values['runtime-service-account'] !== undefined ||
|
|
values['rehome-director-service-account'] !== undefined ||
|
|
values['rehome-audience'] !== undefined ||
|
|
values['expected-rehome-generation'] !== undefined ||
|
|
values['rehome-control-origin'] !== undefined ||
|
|
values['region-correction-cohort-percent'] !== undefined ||
|
|
values['reserve-placement'] !== undefined ||
|
|
values['shadow-seat-feed-cells'] !== undefined
|
|
) {
|
|
throw new Error('director configuration arguments require --role director')
|
|
}
|
|
}
|
|
if (
|
|
values['reserve-placement'] !== undefined &&
|
|
!['preserve', ...RESERVE_PLACEMENT_MODES].includes(values['reserve-placement'])
|
|
) {
|
|
throw new Error('--reserve-placement must be preserve, off, dry-run or on')
|
|
}
|
|
if (
|
|
values.role === 'director' &&
|
|
values['capacity-cell-id'] !== undefined &&
|
|
values['director-cells-json'] === undefined
|
|
) {
|
|
throw new Error('--capacity-cell-id requires --director-cells-json')
|
|
}
|
|
if (values['regional-placement-enabled'] !== undefined) {
|
|
throw new Error('regional placement changes use the audited runtime-setting step')
|
|
}
|
|
if (
|
|
values['regional-placement-secret-version'] !== undefined &&
|
|
!/^[1-9][0-9]*$/.test(values['regional-placement-secret-version'])
|
|
) {
|
|
throw new Error('--regional-placement-secret-version must be a positive integer')
|
|
}
|
|
if (!['true', 'false'].includes(values['prune-revisions'] ?? 'false')) {
|
|
throw new Error('--prune-revisions must be true or false')
|
|
}
|
|
if (!['true', 'false'].includes(values['bootstrap-runtime-identity'] ?? 'false')) {
|
|
throw new Error('--bootstrap-runtime-identity must be true or false')
|
|
}
|
|
if (values.role === 'director') {
|
|
directorDeploymentEnvironment(values)
|
|
projectServiceAccount(values, 'runtime-service-account')
|
|
projectServiceAccount(values, 'predecessor-runtime-service-account')
|
|
if (
|
|
values['bootstrap-runtime-identity'] === 'true' &&
|
|
(!values['runtime-service-account'] ||
|
|
!values['predecessor-runtime-service-account'] ||
|
|
!values['predecessor-image-digest'])
|
|
) {
|
|
throw new Error('runtime identity bootstrap requires exact predecessor digest and identities')
|
|
}
|
|
if (
|
|
values['predecessor-image-digest'] !== undefined &&
|
|
!/^sha256:[a-f0-9]{64}$/.test(values['predecessor-image-digest'])
|
|
) {
|
|
throw new Error('--predecessor-image-digest must be an immutable digest')
|
|
}
|
|
const rehomePair = [
|
|
'rehome-director-service-account',
|
|
'rehome-audience'
|
|
].map((key) => values[key] !== undefined)
|
|
if (rehomePair[0] !== rehomePair[1]) {
|
|
throw new Error('rehome identity and audience must be configured together')
|
|
}
|
|
if (values['rehome-audience'] !== undefined) {
|
|
const audience = new URL(values['rehome-audience'])
|
|
if (
|
|
audience.protocol !== 'https:' ||
|
|
audience.pathname !== '/v1/admin/host-drain' ||
|
|
audience.search ||
|
|
audience.hash
|
|
) {
|
|
throw new Error('--rehome-audience must be an exact host-drain HTTPS URL')
|
|
}
|
|
}
|
|
// --admin-audience alone serves the reserve guard's reads; durable verification needs all three.
|
|
const controlArguments = ['expected-rehome-generation', 'rehome-control-origin'].map(
|
|
(key) => values[key] !== undefined
|
|
)
|
|
if (
|
|
controlArguments.some(Boolean) &&
|
|
(!controlArguments.every(Boolean) || values['admin-audience'] === undefined)
|
|
) {
|
|
throw new Error('durable rehome verification arguments must be configured together')
|
|
}
|
|
if (
|
|
values['expected-rehome-generation'] !== undefined &&
|
|
!/^(0|[1-9][0-9]*)$/.test(values['expected-rehome-generation'])
|
|
) {
|
|
throw new Error('--expected-rehome-generation must be a nonnegative integer')
|
|
}
|
|
if (values['rehome-control-origin'] !== undefined) {
|
|
const origin = new URL(values['rehome-control-origin'])
|
|
if (origin.protocol !== 'https:' || origin.origin !== values['rehome-control-origin']) {
|
|
throw new Error('--rehome-control-origin must be an HTTPS origin')
|
|
}
|
|
}
|
|
}
|
|
return values
|
|
}
|
|
|
|
function commandJson(args) {
|
|
const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
|
|
if (result.status !== 0) {
|
|
throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`)
|
|
}
|
|
return JSON.parse(result.stdout)
|
|
}
|
|
|
|
function commandText(args, { sensitive = false } = {}) {
|
|
const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
|
|
if (result.status !== 0) {
|
|
const detail = sensitive ? 'credential command failed' : result.stderr.trim()
|
|
throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${detail}`)
|
|
}
|
|
return result.stdout.trim()
|
|
}
|
|
|
|
export function suppliedAdminIdentityToken(environment = process.env) {
|
|
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
|
if (token === undefined) return null
|
|
// The workflow supplies a masked Google ID token because external-account gcloud cannot mint one directly.
|
|
if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
|
|
throw new Error('invalid supplied admin identity token')
|
|
}
|
|
return token
|
|
}
|
|
|
|
// The unverified `email` claim, only to name the caller in an error; the director verifies it.
|
|
export function adminTokenEmail(token) {
|
|
try {
|
|
const payload = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString('utf8'))
|
|
return typeof payload?.email === 'string' ? payload.email : undefined
|
|
} catch {
|
|
return undefined
|
|
}
|
|
}
|
|
|
|
// cell-admit-mode answers only the revision's deploy identity, so a drifted one (a revision that
|
|
// trusts another account than the workflow's) fails here by name instead of as a bare 401.
|
|
export function assertDirectorTrustsAdminCaller(token, environment, label) {
|
|
const trusted = environment.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT
|
|
const caller = adminTokenEmail(token)
|
|
if (trusted === undefined || caller === undefined || caller === trusted) return
|
|
throw new Error(
|
|
`the ${label} director trusts ${trusted} for deploy admin reads, but this deploy authenticates as ${caller}; ` +
|
|
'run as that identity or correct the revision\'s ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT'
|
|
)
|
|
}
|
|
|
|
function adminIdentityToken(config) {
|
|
const supplied = suppliedAdminIdentityToken()
|
|
if (supplied === null && !config['admin-audience']) {
|
|
throw new Error('admin reads need --admin-audience or a supplied ORCA_RELAY_ADMIN_ID_TOKEN')
|
|
}
|
|
return (
|
|
supplied ??
|
|
commandText(['auth', 'print-identity-token', `--audiences=${config['admin-audience']}`], {
|
|
sensitive: true
|
|
})
|
|
)
|
|
}
|
|
|
|
function serviceArguments(config) {
|
|
return ['--project', config.project, '--region', config.region]
|
|
}
|
|
|
|
export function directorStartupProbeArguments(role) {
|
|
return role === 'director' ? ['--startup-probe', DIRECTOR_STARTUP_PROBE] : []
|
|
}
|
|
|
|
function describeService(config) {
|
|
return commandJson([
|
|
'run',
|
|
'services',
|
|
'describe',
|
|
config.service,
|
|
...serviceArguments(config),
|
|
'--format=json'
|
|
])
|
|
}
|
|
|
|
function describeRevision(config, revision) {
|
|
return commandJson([
|
|
'run',
|
|
'revisions',
|
|
'describe',
|
|
revision,
|
|
...serviceArguments(config),
|
|
'--format=json'
|
|
])
|
|
}
|
|
|
|
function listRevisions(config) {
|
|
return commandJson([
|
|
'run',
|
|
'revisions',
|
|
'list',
|
|
'--service',
|
|
config.service,
|
|
...serviceArguments(config),
|
|
'--format=json'
|
|
])
|
|
}
|
|
|
|
function deleteRevision(config, revision) {
|
|
commandText([
|
|
'run',
|
|
'revisions',
|
|
'delete',
|
|
revision,
|
|
...serviceArguments(config),
|
|
'--quiet'
|
|
])
|
|
}
|
|
|
|
function updateTraffic(config, args) {
|
|
commandText([
|
|
'run',
|
|
'services',
|
|
'update-traffic',
|
|
config.service,
|
|
...serviceArguments(config),
|
|
...args,
|
|
'--quiet'
|
|
])
|
|
}
|
|
|
|
function removeDirectorTrafficTags(config, operations, retained = new Set()) {
|
|
const tags = trafficTags(operations.describeService(config)).filter(
|
|
(tag) => !retained.has(tag)
|
|
)
|
|
if (tags.length === 0) return
|
|
try {
|
|
operations.updateTraffic(config, [`--remove-tags=${tags.join(',')}`])
|
|
} catch (error) {
|
|
const remaining = new Set(trafficTags(operations.describeService(config)))
|
|
if (tags.some((tag) => remaining.has(tag))) throw error
|
|
}
|
|
}
|
|
|
|
function pruneDirectorRevisions(config, operations) {
|
|
const service = operations.describeService(config)
|
|
const retained = new Set([
|
|
activeRevision(service),
|
|
taggedTraffic(service, SELECTOR_ROLLBACK_TAG).revision
|
|
])
|
|
for (const revision of operations.listRevisions(config)) {
|
|
const name = revision.metadata?.name
|
|
if (!name) throw new Error('director revision list contains an unnamed revision')
|
|
if (!retained.has(name)) operations.deleteRevision(config, name)
|
|
}
|
|
const remaining = operations
|
|
.listRevisions(config)
|
|
.map((revision) => revision.metadata?.name)
|
|
if (
|
|
remaining.length !== retained.size ||
|
|
remaining.some((name) => !name || !retained.has(name))
|
|
) {
|
|
throw new Error('old director revisions remain after deployment')
|
|
}
|
|
}
|
|
|
|
function deployCandidate(
|
|
config,
|
|
tag,
|
|
env = {},
|
|
image = config.image,
|
|
minInstances = config['min-instances'],
|
|
maxInstances,
|
|
regionalPlacementVersion
|
|
) {
|
|
const args = [
|
|
'run',
|
|
'services',
|
|
'update',
|
|
config.service,
|
|
...serviceArguments(config),
|
|
'--image',
|
|
image,
|
|
'--tag',
|
|
tag,
|
|
'--no-traffic'
|
|
]
|
|
if (maxInstances !== undefined) args.push('--max', String(maxInstances))
|
|
if (config.role === 'director') {
|
|
args.push(
|
|
'--update-secrets',
|
|
`${DIRECTOR_REGIONAL_PLACEMENT_ENV}=${DIRECTOR_REGIONAL_PLACEMENT_SECRET}:${regionalPlacementVersion}`
|
|
)
|
|
if (config['runtime-service-account'] !== undefined) {
|
|
args.push('--service-account', config['runtime-service-account'])
|
|
}
|
|
}
|
|
args.push(...directorStartupProbeArguments(config.role))
|
|
if (minInstances !== undefined) {
|
|
args.push('--min-instances', String(minInstances))
|
|
}
|
|
const entries = Object.entries(env)
|
|
if (entries.length > 0) {
|
|
args.push('--update-env-vars', environmentUpdateValue(env))
|
|
}
|
|
args.push('--quiet')
|
|
commandText(args)
|
|
}
|
|
|
|
function revisionShape(revision, mutableEnvironment, allowServiceAccountChange = false) {
|
|
const spec = structuredClone(revision.spec ?? {})
|
|
if (allowServiceAccountChange) delete spec.serviceAccountName
|
|
for (const container of spec.containers ?? []) {
|
|
delete container.image
|
|
delete container.startupProbe
|
|
container.env = (container.env ?? []).filter(
|
|
({ name }) => !(name in mutableEnvironment)
|
|
)
|
|
}
|
|
const ignoredAnnotations = new Set([
|
|
'autoscaling.knative.dev/minScale',
|
|
'run.googleapis.com/client-name',
|
|
'run.googleapis.com/client-version',
|
|
'run.googleapis.com/operation-id',
|
|
'serving.knative.dev/creator'
|
|
])
|
|
const annotations = Object.fromEntries(
|
|
Object.entries(revision.metadata?.annotations ?? {}).filter(
|
|
([name]) => !ignoredAnnotations.has(name)
|
|
)
|
|
)
|
|
return { spec, annotations }
|
|
}
|
|
|
|
function assertPreservedRevisionShape(
|
|
serving,
|
|
candidate,
|
|
mutableEnvironment,
|
|
allowServiceAccountChange = false
|
|
) {
|
|
if (!isDeepStrictEqual(
|
|
revisionShape(serving, mutableEnvironment, allowServiceAccountChange),
|
|
revisionShape(candidate, mutableEnvironment, allowServiceAccountChange)
|
|
)) {
|
|
throw new Error('director candidate changed unrelated revision shape')
|
|
}
|
|
}
|
|
|
|
function releaseLabel(releaseId) {
|
|
const normalized = releaseId.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
|
if (!normalized) throw new Error('release id has no usable characters')
|
|
return normalized.slice(0, 32).replace(/-$/g, '')
|
|
}
|
|
|
|
export function cloudRunTrafficTag(service, prefix, releaseId) {
|
|
if (!/^[a-z][a-z0-9-]*$/.test(service)) throw new Error('invalid Cloud Run service name')
|
|
if (!/^[a-z][a-z0-9-]*$/.test(prefix)) throw new Error('invalid Cloud Run tag prefix')
|
|
const digest = createHash('sha256').update(releaseLabel(releaseId)).digest('hex').slice(0, 9)
|
|
const tag = `${prefix}-${digest}`
|
|
// Cloud Run imposes this combined bound in addition to the standalone tag bound.
|
|
if (service.length + tag.length > 46) throw new Error('Cloud Run service leaves no safe tag space')
|
|
return tag
|
|
}
|
|
|
|
function cellIdentifier(sourceCellId, releaseId) {
|
|
const suffix = releaseLabel(releaseId)
|
|
const available = 128 - suffix.length - 2
|
|
return `${sourceCellId.slice(0, available)}--${suffix}`
|
|
}
|
|
|
|
async function waitForHealth(origin, connectionCapacityProtocol) {
|
|
const response = await fetch(`${origin}/health`, { signal: AbortSignal.timeout(15_000) })
|
|
const body = await response.json()
|
|
if (
|
|
!response.ok ||
|
|
body.ok !== true ||
|
|
(connectionCapacityProtocol !== undefined &&
|
|
body.connectionCapacityProtocol !== connectionCapacityProtocol)
|
|
) {
|
|
throw new Error(`candidate health failed at ${origin}`)
|
|
}
|
|
}
|
|
|
|
export async function waitForEvacuationCapacity(
|
|
adminPost,
|
|
sourceCellId,
|
|
targetCellId,
|
|
{ pollIntervalMs = POLL_INTERVAL_MS, timeoutMs = MIGRATION_TIMEOUT_MS } = {}
|
|
) {
|
|
const deadline = Date.now() + timeoutMs
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
return await adminPost('/v1/admin/evacuation-capacity', {
|
|
v: 1,
|
|
sourceCellId,
|
|
targetCellId
|
|
})
|
|
} catch (error) {
|
|
if (!(error instanceof Error) || !error.message.endsWith(': target_cell_unavailable')) throw error
|
|
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs))
|
|
}
|
|
}
|
|
throw new Error('timed out waiting for candidate cell readiness')
|
|
}
|
|
|
|
async function adminClient(config) {
|
|
const token = adminIdentityToken(config)
|
|
return async (path, body) => {
|
|
const response = await fetch(`${config['director-origin']}${path}`, {
|
|
method: 'POST',
|
|
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
|
body: JSON.stringify(body),
|
|
signal: AbortSignal.timeout(30_000)
|
|
})
|
|
const result = await response.json().catch(() => ({ error: `http_${response.status}` }))
|
|
if (!response.ok) throw new Error(`${path} failed: ${result.error ?? response.status}`)
|
|
return result
|
|
}
|
|
}
|
|
|
|
export async function assertRegionalRehomeDisabled(
|
|
config,
|
|
origin,
|
|
fetchImpl = fetch
|
|
) {
|
|
const response = await fetchImpl(`${origin}/v1/admin/regional-rehome-control`, {
|
|
method: 'POST',
|
|
headers: {
|
|
authorization: `Bearer ${adminIdentityToken(config)}`,
|
|
'content-type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ v: 1, action: 'inspect' }),
|
|
signal: AbortSignal.timeout(30_000)
|
|
})
|
|
const result = await response.json().catch(() => ({ error: `http_${response.status}` }))
|
|
if (!response.ok) {
|
|
throw new Error(`regional rehome inspection failed: ${result.error ?? response.status}`)
|
|
}
|
|
const generation = Number(config['expected-rehome-generation'])
|
|
if (
|
|
result.v !== 1 ||
|
|
result.control?.enabled !== false ||
|
|
result.control?.generation !== generation
|
|
) {
|
|
throw new Error('regional rehome control is not durably disabled at the expected generation')
|
|
}
|
|
return result.control
|
|
}
|
|
|
|
// `undefined` when the director predates the reserve census (its router 404s the route).
|
|
export async function readReserveCells(config, origin, cellIds, fetchImpl = fetch) {
|
|
const reserve = []
|
|
for (const cellId of cellIds) {
|
|
const response = await fetchImpl(`${origin}/v1/admin/cell-admit-mode`, {
|
|
method: 'POST',
|
|
headers: {
|
|
authorization: `Bearer ${adminIdentityToken(config)}`,
|
|
'content-type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ v: 1, cellId }),
|
|
signal: AbortSignal.timeout(30_000)
|
|
})
|
|
const result = await response.json().catch(() => null)
|
|
if (response.status === 404 && result === null) return undefined
|
|
if (!response.ok || result?.v !== 1 || !['db', 'reserve'].includes(result.admitMode)) {
|
|
throw new Error(`cell admit mode read failed for ${cellId}: ${result?.error ?? response.status}`)
|
|
}
|
|
if (result.admitMode === 'reserve') reserve.push(cellId)
|
|
}
|
|
return reserve
|
|
}
|
|
|
|
// A director without the census places hosts on reserve cells from stale rows, so traffic may
|
|
// move between revisions that differ in it only while every cell is db.
|
|
export function assertReserveCompatibleTrafficMove(servingReserve, candidateReserve) {
|
|
if ((servingReserve === undefined) === (candidateReserve === undefined)) return
|
|
const reserve = servingReserve ?? candidateReserve
|
|
if (reserve.length > 0) {
|
|
throw new Error(
|
|
`director revisions differ in reserve-mode support while cells are reserve (${reserve.join(', ')}); flip them to db first`
|
|
)
|
|
}
|
|
}
|
|
|
|
// With placement off no director books a reserve cell, so each one's dead-man trips (60-120 s)
|
|
// and every host re-registers at once: the cells go to db first, on purpose.
|
|
export function assertReservePlacementKept(servingPlacement, candidatePlacement, reserve) {
|
|
if (servingPlacement !== 'on' || candidatePlacement === 'on') return
|
|
if (reserve !== undefined && reserve.length > 0) {
|
|
throw new Error(
|
|
`reserve placement would turn ${candidatePlacement} while cells are reserve (${reserve.join(', ')}); flip them to db first`
|
|
)
|
|
}
|
|
}
|
|
|
|
// Placement as the director built it (runtime-status), not as its env asks; an image from before
|
|
// the field reports nothing, and only its env can say.
|
|
export async function readDirectorPlacement(config, origin, fetchImpl = fetch) {
|
|
const response = await fetchImpl(`${origin}/v1/admin/runtime-status`, {
|
|
method: 'POST',
|
|
headers: {
|
|
authorization: `Bearer ${adminIdentityToken(config)}`,
|
|
'content-type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ v: 1 }),
|
|
signal: AbortSignal.timeout(30_000)
|
|
})
|
|
const result = await response.json().catch(() => null)
|
|
if (!response.ok || result?.v !== 1 || result.role !== 'director') {
|
|
throw new Error(`director runtime status failed at ${origin}: ${result?.error ?? response.status}`)
|
|
}
|
|
if (result.reservePlacement === undefined) return 'unreported'
|
|
if (!RESERVE_PLACEMENT_MODES.includes(result.reservePlacement)) {
|
|
throw new Error(`director at ${origin} reports reserve placement ${result.reservePlacement}`)
|
|
}
|
|
return result.reservePlacement
|
|
}
|
|
|
|
const shadowCellSet = (value) =>
|
|
(value ?? '').trim() === 'all'
|
|
? 'all'
|
|
: new Set(
|
|
(value ?? '')
|
|
.split(',')
|
|
.map((cell) => cell.trim())
|
|
.filter(Boolean)
|
|
)
|
|
|
|
// A reserve cell the directors stop polling gets no booking and no reserver poll: its dead-man
|
|
// trips. Shrinking the shadow-cell list past one is refused until that cell is db.
|
|
export function assertShadowCellsKeepReserveCells(servingCells, candidateCells, reserve) {
|
|
if (reserve === undefined || reserve.length === 0) return
|
|
const before = shadowCellSet(servingCells)
|
|
const after = shadowCellSet(candidateCells)
|
|
if (after === 'all') return
|
|
const dropped = reserve.filter((cell) => (before === 'all' || before.has(cell)) && !after.has(cell))
|
|
if (dropped.length > 0) {
|
|
throw new Error(
|
|
`the shadow seat-feed cells would drop reserve cells (${dropped.join(', ')}); flip them to db first`
|
|
)
|
|
}
|
|
}
|
|
|
|
function directorCellIds(...environments) {
|
|
const ids = new Set()
|
|
for (const environment of environments) {
|
|
for (const cell of JSON.parse(environment.ORCA_RELAY_CELLS_JSON ?? '[]')) ids.add(cell.id)
|
|
}
|
|
return [...ids]
|
|
}
|
|
|
|
function assertDirectorRevisionIdentity(revision, config) {
|
|
const expectedRuntimeServiceAccount = config['runtime-service-account']
|
|
if (
|
|
expectedRuntimeServiceAccount !== undefined &&
|
|
revision.spec?.serviceAccountName !== expectedRuntimeServiceAccount
|
|
) {
|
|
throw new Error('director revision uses an unexpected runtime service account')
|
|
}
|
|
}
|
|
|
|
export async function deployDirector(config, tag, overrides = {}) {
|
|
const operations = {
|
|
deployCandidate,
|
|
describeService,
|
|
describeRevision,
|
|
listRevisions,
|
|
deleteRevision,
|
|
updateTraffic,
|
|
waitForHealth,
|
|
assertRegionalRehomeDisabled,
|
|
readReserveCells,
|
|
readDirectorPlacement,
|
|
adminIdentityToken,
|
|
...overrides
|
|
}
|
|
// Why: gcloud does not carry minScale onto a new revision, and the candidate below takes
|
|
// 100% of traffic. Inheriting the serving floor keeps one owner for the number instead of
|
|
// restating it here; public admission is per-instance, so losing it shrinks fleet capacity.
|
|
const initialService = operations.describeService(config)
|
|
const servingRevision = operations.describeRevision(config, activeRevision(initialService))
|
|
const bootstrapRuntimeIdentity = config['bootstrap-runtime-identity'] === 'true'
|
|
if (bootstrapRuntimeIdentity) {
|
|
if (
|
|
servingRevision.spec?.serviceAccountName !==
|
|
config['predecessor-runtime-service-account']
|
|
) {
|
|
throw new Error('director predecessor runtime service account does not match')
|
|
}
|
|
if (
|
|
config['predecessor-runtime-service-account'] === config['runtime-service-account']
|
|
) {
|
|
throw new Error('director runtime identity bootstrap has already completed')
|
|
}
|
|
const predecessorDigest = servingRevision.spec?.containers?.[0]?.image?.split('@').at(-1)
|
|
if (predecessorDigest !== config['predecessor-image-digest']) {
|
|
throw new Error('director predecessor image digest does not match')
|
|
}
|
|
} else {
|
|
assertDirectorRevisionIdentity(servingRevision, config)
|
|
}
|
|
const servingMinimumInstances = revisionMinimumInstances(servingRevision)
|
|
const servingMaximumInstances = revisionMaximumInstances(servingRevision)
|
|
const requiredMaximumInstances = config['max-instances'] === undefined
|
|
? servingMaximumInstances
|
|
: Number(config['max-instances'])
|
|
if (servingMaximumInstances !== requiredMaximumInstances) {
|
|
throw new Error(
|
|
`serving revision holds ${servingMaximumInstances} maximum instances, expected ${requiredMaximumInstances}`
|
|
)
|
|
}
|
|
const servingSecrets = revisionSecretEnvironment(servingRevision)
|
|
const servingRegionalPlacementVersion =
|
|
servingSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version
|
|
const targetRegionalPlacementVersion =
|
|
config['regional-placement-secret-version'] ?? servingRegionalPlacementVersion
|
|
if (!/^[1-9][0-9]*$/.test(targetRegionalPlacementVersion ?? '')) {
|
|
throw new Error('director deployment requires an exact regional placement secret version')
|
|
}
|
|
const rollbackRegionalPlacementVersion =
|
|
/^[1-9][0-9]*$/.test(servingRegionalPlacementVersion ?? '')
|
|
? servingRegionalPlacementVersion
|
|
: targetRegionalPlacementVersion
|
|
const requiredMinimumInstances =
|
|
config['min-instances'] === undefined
|
|
? servingMinimumInstances
|
|
: Number(config['min-instances'])
|
|
const requiredCapacityProtocol =
|
|
config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined
|
|
const currentEnvironment = revisionEnvironment(servingRevision)
|
|
const deploymentEnvironment = directorDeploymentEnvironment(config)
|
|
deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent(
|
|
currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0'
|
|
)
|
|
if (currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] !== undefined) {
|
|
deploymentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] ??= currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV]
|
|
}
|
|
if (currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] !== undefined) {
|
|
deploymentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] ??= currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]
|
|
}
|
|
if (config['region-correction-cohort-percent'] !== undefined &&
|
|
config['region-correction-cohort-percent'] !== 'preserve' &&
|
|
config['expected-rehome-generation'] === undefined) {
|
|
throw new Error('cohort changes require an exact disabled regional-rehome generation')
|
|
}
|
|
const mutableEnvironment = {
|
|
...deploymentEnvironment,
|
|
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: ''
|
|
}
|
|
const topology = config['director-cells-json'] === undefined
|
|
? { changed: false }
|
|
: config['capacity-cell-id'] === undefined
|
|
? directorCellSetAddition(
|
|
currentEnvironment.ORCA_RELAY_CELLS_JSON,
|
|
deploymentEnvironment.ORCA_RELAY_CELLS_JSON
|
|
)
|
|
: directorTopologyChange(
|
|
currentEnvironment.ORCA_RELAY_CELLS_JSON,
|
|
deploymentEnvironment.ORCA_RELAY_CELLS_JSON,
|
|
config['capacity-cell-id']
|
|
)
|
|
const verifyRehomeDisabled = async (origin) => {
|
|
if (config['expected-rehome-generation'] === undefined) return
|
|
await operations.assertRegionalRehomeDisabled(config, origin)
|
|
}
|
|
if (!bootstrapRuntimeIdentity) {
|
|
await verifyRehomeDisabled(config['rehome-control-origin'])
|
|
}
|
|
removeDirectorTrafficTags(config, operations)
|
|
let deployed
|
|
let promoted = false
|
|
try {
|
|
operations.deployCandidate(
|
|
config,
|
|
SELECTOR_ROLLBACK_TAG,
|
|
deploymentEnvironment,
|
|
config.image,
|
|
0,
|
|
requiredMaximumInstances,
|
|
rollbackRegionalPlacementVersion
|
|
)
|
|
const rollback = taggedTraffic(
|
|
operations.describeService(config),
|
|
SELECTOR_ROLLBACK_TAG
|
|
)
|
|
const rollbackRevision = operations.describeRevision(config, rollback.revision)
|
|
assertDirectorRevisionIdentity(rollbackRevision, config)
|
|
assertPreservedRevisionShape(
|
|
servingRevision,
|
|
rollbackRevision,
|
|
mutableEnvironment,
|
|
bootstrapRuntimeIdentity
|
|
)
|
|
const rollbackEnvironment = revisionEnvironment(rollbackRevision)
|
|
const rollbackSecrets = revisionSecretEnvironment(rollbackRevision)
|
|
if (
|
|
rollbackEnvironment.ORCA_RELAY_ROLE !== 'director' ||
|
|
rollbackEnvironment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !==
|
|
SELECTOR_REVISION_MARKER ||
|
|
!hasExpectedEnvironment(rollbackEnvironment, deploymentEnvironment) ||
|
|
rollbackSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.secret !==
|
|
DIRECTOR_REGIONAL_PLACEMENT_SECRET ||
|
|
rollbackSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version !==
|
|
rollbackRegionalPlacementVersion ||
|
|
revisionMinimumInstances(rollbackRevision) !== 0
|
|
) {
|
|
throw new Error('rollback revision is not selector-compatible')
|
|
}
|
|
await operations.waitForHealth(rollback.origin, requiredCapacityProtocol)
|
|
await verifyRehomeDisabled(rollback.origin)
|
|
operations.deployCandidate(
|
|
config,
|
|
tag,
|
|
deploymentEnvironment,
|
|
config.image,
|
|
requiredMinimumInstances,
|
|
requiredMaximumInstances,
|
|
targetRegionalPlacementVersion
|
|
)
|
|
const candidate = taggedTraffic(operations.describeService(config), tag)
|
|
const candidateRevision = operations.describeRevision(config, candidate.revision)
|
|
assertDirectorRevisionIdentity(candidateRevision, config)
|
|
assertPreservedRevisionShape(
|
|
servingRevision,
|
|
candidateRevision,
|
|
mutableEnvironment,
|
|
bootstrapRuntimeIdentity
|
|
)
|
|
const environment = revisionEnvironment(candidateRevision)
|
|
const secrets = revisionSecretEnvironment(candidateRevision)
|
|
if (
|
|
environment.ORCA_RELAY_ROLE !== 'director' ||
|
|
environment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== SELECTOR_REVISION_MARKER ||
|
|
!hasExpectedEnvironment(environment, deploymentEnvironment) ||
|
|
secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.secret !==
|
|
DIRECTOR_REGIONAL_PLACEMENT_SECRET ||
|
|
secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version !==
|
|
targetRegionalPlacementVersion
|
|
) {
|
|
throw new Error('stable relay service is not selector-compatible')
|
|
}
|
|
// Fail before the traffic move, so a candidate that lost the floor never serves.
|
|
const candidateMinimumInstances = revisionMinimumInstances(candidateRevision)
|
|
if (candidateMinimumInstances !== requiredMinimumInstances) {
|
|
throw new Error(
|
|
`candidate holds ${candidateMinimumInstances} minimum instances, expected ${requiredMinimumInstances}`
|
|
)
|
|
}
|
|
await operations.waitForHealth(candidate.origin, requiredCapacityProtocol)
|
|
await verifyRehomeDisabled(candidate.origin)
|
|
// The same image and the same placement and shadow-cell settings support the same things;
|
|
// anything else needs every cell's admit mode (and the admin token that reads it).
|
|
const sameImage =
|
|
servingRevision.spec?.containers?.[0]?.image?.split('@').at(-1) === config.image?.split('@').at(-1)
|
|
const sameSetting = (name) => (currentEnvironment[name] ?? '') === (environment[name] ?? '')
|
|
if (sameImage && sameSetting(DIRECTOR_RESERVE_PLACEMENT_ENV) && sameSetting(DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV)) {
|
|
console.warn(JSON.stringify({ event: 'director_reserve_support_checked', skipped: 'same image and settings' }))
|
|
} else {
|
|
const servingOrigin = initialService.status?.url
|
|
if (!servingOrigin) throw new Error('relay service reports no URL for the serving revision')
|
|
const token = operations.adminIdentityToken(config)
|
|
assertDirectorTrustsAdminCaller(token, currentEnvironment, 'serving')
|
|
assertDirectorTrustsAdminCaller(token, environment, 'candidate')
|
|
const cellIds = directorCellIds(currentEnvironment, environment)
|
|
const servingReserve = await operations.readReserveCells(config, servingOrigin, cellIds)
|
|
const candidateReserve = await operations.readReserveCells(config, candidate.origin, cellIds)
|
|
const reserve = candidateReserve ?? servingReserve
|
|
// Placement as each director built it, which an env string alone does not say.
|
|
const placementAt = async (origin, env) => {
|
|
const placement = await operations.readDirectorPlacement(config, origin)
|
|
return placement === 'unreported' ? (env[DIRECTOR_RESERVE_PLACEMENT_ENV] ?? 'off') : placement
|
|
}
|
|
const servingPlacement = await placementAt(servingOrigin, currentEnvironment)
|
|
const candidatePlacement = await placementAt(candidate.origin, environment)
|
|
assertReserveCompatibleTrafficMove(servingReserve, candidateReserve)
|
|
assertReservePlacementKept(servingPlacement, candidatePlacement, reserve)
|
|
assertShadowCellsKeepReserveCells(
|
|
currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV],
|
|
environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV],
|
|
reserve
|
|
)
|
|
console.warn(
|
|
JSON.stringify({
|
|
event: 'director_reserve_support_checked',
|
|
reservePlacement: { serving: servingPlacement, candidate: candidatePlacement },
|
|
serving: servingReserve === undefined ? 'unsupported' : servingReserve,
|
|
candidate: candidateReserve === undefined ? 'unsupported' : candidateReserve
|
|
})
|
|
)
|
|
}
|
|
operations.updateTraffic(config, [`--to-tags=${tag}=100`])
|
|
promoted = true
|
|
removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG]))
|
|
deployed = {
|
|
event: 'director_deployed',
|
|
revision: candidate.revision,
|
|
rollbackRevision: rollback.revision,
|
|
topologyChanged: topology.changed
|
|
}
|
|
} catch (error) {
|
|
const recoveryErrors = [error]
|
|
if (promoted) {
|
|
try {
|
|
operations.updateTraffic(config, [`--to-tags=${SELECTOR_ROLLBACK_TAG}=100`])
|
|
} catch (rollbackError) {
|
|
recoveryErrors.push(rollbackError)
|
|
}
|
|
}
|
|
try {
|
|
removeDirectorTrafficTags(
|
|
config,
|
|
operations,
|
|
promoted ? new Set([SELECTOR_ROLLBACK_TAG]) : new Set()
|
|
)
|
|
} catch (cleanupError) {
|
|
recoveryErrors.push(cleanupError)
|
|
}
|
|
if (recoveryErrors.length > 1) {
|
|
const details = recoveryErrors
|
|
.map((failure) => failure instanceof Error ? failure.message : String(failure))
|
|
.join('; ')
|
|
throw new AggregateError(recoveryErrors, `director deploy recovery failed: ${details}`)
|
|
}
|
|
throw error
|
|
}
|
|
if (config['prune-revisions'] === 'true') pruneDirectorRevisions(config, operations)
|
|
process.stdout.write(`${JSON.stringify(deployed)}\n`)
|
|
}
|
|
|
|
async function waitForTargetRegistration(adminPost, sourceCellId, targetCellId) {
|
|
const deadline = Date.now() + MIGRATION_TIMEOUT_MS
|
|
while (Date.now() < deadline) {
|
|
const status = await adminPost('/v1/admin/evacuation-status', {
|
|
v: 1,
|
|
sourceCellId,
|
|
targetCellId,
|
|
completeReady: false
|
|
})
|
|
process.stdout.write(
|
|
`${JSON.stringify({ event: 'migration_registration', ...status })}\n`
|
|
)
|
|
if (status.inProgress === status.targetRegistered) return
|
|
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
|
|
}
|
|
throw new Error('timed out waiting for target control registrations')
|
|
}
|
|
|
|
async function waitForCompletion(adminPost, sourceCellId, targetCellId) {
|
|
const deadline = Date.now() + MIGRATION_TIMEOUT_MS
|
|
while (Date.now() < deadline) {
|
|
const status = await adminPost('/v1/admin/evacuation-status', {
|
|
v: 1,
|
|
sourceCellId,
|
|
targetCellId,
|
|
completeReady: true
|
|
})
|
|
process.stdout.write(`${JSON.stringify({ event: 'migration_completion', ...status })}\n`)
|
|
if (status.inProgress === 0) return
|
|
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
|
|
}
|
|
throw new Error('timed out waiting for source activity to drain')
|
|
}
|
|
|
|
async function startAllEvacuations(adminPost, sourceCellId, targetCellId) {
|
|
for (;;) {
|
|
const result = await adminPost('/v1/admin/evacuate-cell', {
|
|
v: 1,
|
|
sourceCellId,
|
|
targetCellId,
|
|
limit: 100
|
|
})
|
|
process.stdout.write(`${JSON.stringify({ event: 'migration_batch', started: result.started })}\n`)
|
|
if (result.started === 0) return
|
|
}
|
|
}
|
|
|
|
async function deployCell(config, tag, oldTag, drainTag) {
|
|
const initialService = describeService(config)
|
|
const currentRevision = activeRevision(initialService)
|
|
const currentRevisionState = describeRevision(config, currentRevision)
|
|
const currentEnv = revisionEnvironment(currentRevisionState)
|
|
const currentImage = currentRevisionState.spec?.containers?.[0]?.image
|
|
const sourceCellId = currentEnv.ORCA_RELAY_CELL_ID
|
|
const sourceOrigin = currentEnv.ORCA_RELAY_CELL_URL
|
|
const capacityRequests = Number(currentEnv.ORCA_RELAY_CELL_CAPACITY)
|
|
if (
|
|
currentEnv.ORCA_RELAY_ROLE !== 'cell' ||
|
|
!sourceCellId ||
|
|
!sourceOrigin ||
|
|
!currentImage ||
|
|
!Number.isInteger(capacityRequests) ||
|
|
capacityRequests <= 0
|
|
) {
|
|
throw new Error('active cell revision has invalid role, identity, image, or capacity')
|
|
}
|
|
updateTraffic(config, [`--update-tags=${drainTag}=${currentRevision}`])
|
|
const drainRevision = taggedTraffic(describeService(config), drainTag)
|
|
const previousOrigin = taggedRevisionOrigin(initialService.status.url, oldTag)
|
|
const candidateOrigin = taggedRevisionOrigin(initialService.status.url, tag)
|
|
const targetCellId = cellIdentifier(sourceCellId, config['release-id'])
|
|
const adminPost = await adminClient(config)
|
|
|
|
await adminPost('/v1/admin/cell-config', {
|
|
v: 1,
|
|
cellId: targetCellId,
|
|
cellUrl: candidateOrigin,
|
|
capacityRequests,
|
|
enabled: false
|
|
})
|
|
deployCandidate(config, tag, {
|
|
ORCA_RELAY_CELL_ID: targetCellId,
|
|
ORCA_RELAY_CELL_URL: candidateOrigin,
|
|
ORCA_RELAY_PUBLIC_URL: candidateOrigin
|
|
})
|
|
const candidate = taggedTraffic(describeService(config), tag)
|
|
if (candidate.origin !== candidateOrigin) {
|
|
throw new Error('queried candidate tag URL mismatches its configured origin')
|
|
}
|
|
await waitForHealth(candidate.origin)
|
|
deployCandidate(
|
|
config,
|
|
oldTag,
|
|
{
|
|
ORCA_RELAY_CELL_ID: sourceCellId,
|
|
ORCA_RELAY_CELL_URL: previousOrigin,
|
|
ORCA_RELAY_PUBLIC_URL: previousOrigin
|
|
},
|
|
currentImage
|
|
)
|
|
const previous = taggedTraffic(describeService(config), oldTag)
|
|
if (previous.origin !== previousOrigin) {
|
|
throw new Error('queried previous tag URL mismatches its configured origin')
|
|
}
|
|
await waitForHealth(previous.origin)
|
|
|
|
// HTTP health precedes the authenticated heartbeat that makes a migration target eligible.
|
|
await waitForEvacuationCapacity(adminPost, sourceCellId, targetCellId)
|
|
await adminPost('/v1/admin/cell-state', { v: 1, cellId: sourceCellId, enabled: false })
|
|
let sourceReconfigured = false
|
|
try {
|
|
const capacity = await adminPost('/v1/admin/evacuation-capacity', {
|
|
v: 1,
|
|
sourceCellId,
|
|
targetCellId
|
|
})
|
|
process.stdout.write(`${JSON.stringify({ event: 'migration_capacity', ...capacity })}\n`)
|
|
if (capacity.requiredTargetUnits > capacity.availableTargetUnits) {
|
|
throw new Error('candidate lacks durable reservation headroom for target-first migration')
|
|
}
|
|
// The keeper uses the old image and cell identity without competing with live controls.
|
|
await adminPost('/v1/admin/cell-config', {
|
|
v: 1,
|
|
cellId: sourceCellId,
|
|
cellUrl: previous.origin,
|
|
capacityRequests,
|
|
enabled: false
|
|
})
|
|
sourceReconfigured = true
|
|
await adminPost('/v1/admin/cell-config', {
|
|
v: 1,
|
|
cellId: targetCellId,
|
|
cellUrl: candidate.origin,
|
|
capacityRequests,
|
|
enabled: true
|
|
})
|
|
} catch (error) {
|
|
if (sourceReconfigured) {
|
|
await adminPost('/v1/admin/cell-config', {
|
|
v: 1,
|
|
cellId: sourceCellId,
|
|
cellUrl: sourceOrigin,
|
|
capacityRequests,
|
|
enabled: true
|
|
})
|
|
} else {
|
|
await adminPost('/v1/admin/cell-state', { v: 1, cellId: sourceCellId, enabled: true })
|
|
}
|
|
throw error
|
|
}
|
|
await startAllEvacuations(adminPost, sourceCellId, targetCellId)
|
|
|
|
await fetch(`${drainRevision.origin}/v1/admin/drain`, {
|
|
method: 'POST',
|
|
headers: {
|
|
authorization: `Bearer ${adminIdentityToken(config)}`,
|
|
'content-type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ v: 1, graceMs: 120_000 }),
|
|
signal: AbortSignal.timeout(30_000)
|
|
}).then(async (response) => {
|
|
if (!response.ok) throw new Error(`old revision drain failed: ${response.status}`)
|
|
})
|
|
|
|
await waitForTargetRegistration(adminPost, sourceCellId, targetCellId)
|
|
updateTraffic(config, [`--to-tags=${tag}=100`])
|
|
await waitForCompletion(adminPost, sourceCellId, targetCellId)
|
|
const finalEnabled = (config['final-admission'] ?? 'enabled') === 'enabled'
|
|
if (!finalEnabled) {
|
|
// GCE-backed staging keeps stamped cells runnable for regression without assigning normal traffic.
|
|
await adminPost('/v1/admin/cell-state', { v: 1, cellId: targetCellId, enabled: false })
|
|
}
|
|
process.stdout.write(
|
|
`${JSON.stringify({
|
|
event: 'cell_deployed',
|
|
service: config.service,
|
|
sourceCellId,
|
|
targetCellId,
|
|
enabled: finalEnabled,
|
|
drainRevision: drainRevision.revision,
|
|
previousRevision: previous.revision,
|
|
candidateRevision: candidate.revision
|
|
})}\n`
|
|
)
|
|
}
|
|
|
|
export async function main(argv = process.argv.slice(2)) {
|
|
const config = parseArguments(argv)
|
|
const tag = cloudRunTrafficTag(config.service, 'candidate', config['release-id'])
|
|
const oldTag = cloudRunTrafficTag(config.service, 'previous', config['release-id'])
|
|
const drainTag = cloudRunTrafficTag(config.service, 'drain', config['release-id'])
|
|
if (config.role === 'director') await deployDirector(config, tag)
|
|
else await deployCell(config, tag, oldTag, drainTag)
|
|
}
|
|
|
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
main().catch((error) => {
|
|
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
|
process.exitCode = 1
|
|
})
|
|
}
|