mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
fix(relay): the cell switch tool changes only the named switches (#27056)
* fix(relay): the cell switch tool changes only the named switches and checks every key on read-back A cell resets a key missing from its switch file to its default, so the tool now reads the current object and changes only what --set names. The read-back compares every known key and fails when the cell reports ignoredKeys. * fix(relay): the cell switch tool refuses a switch the cell's image does not support It reads supportedFlags from runtime-status (#27057). An image from before that field takes only readinessLocal and ticketCheck off/shadow, so ticketCheck=enforce cannot void an old cell's object. * fix(relay): the switch tool validates the whole object, refuses voided files, and records admitMode in Postgres Every key and value of the object to be written is checked against what the image accepts (an image without the list: only the keys it applies, ticketCheck off/shadow). A file the cell did not apply is refused. admitMode=reserve needs a typed RESERVE <cell>; Postgres records reserve before the file is written, and db only after the cell reports every control leased again. * feat(relay): the switch tool knows reregisterInFlight (1-16), the flip-back pace * fix(relay): the switch tool closes the cache gap before reserve, undoes a refused write, and has a break-glass db flip After recording reserve in Postgres it waits past every director's 5 s cache and records it again (re-running the open-flow check) before writing the file. A file write refused outright (412 or 4xx) puts Postgres back to db; after a 5xx it stays reserve. An object the cell did not apply is repaired when the new object passes the image check. Break glass (typed BREAK GLASS <cell>) records db for a dead or unreachable reserve cell without its read-back. * fix(relay): a director deploy refuses a traffic move across reserve-mode support while any cell is reserve * fix(relay): the flip-back wait scales with the cell's controls and pace, and says how to finish * fix(relay): the switch tool refuses to re-arm a tripped cell, breaks glass only on a silent one, and undoes reserve if the second post is refused * feat(relay): director deploys set reserve placement, and refuse turning it off while any cell is reserve * fix(relay): Terraform keeps the deploy-owned placement and shadow cells, and the deploy guard reads placement as built and refuses dropping a reserve cell from the shadow list * fix(relay): director deploys take --admin-audience alone, name a drifted deploy identity, and carry the shadow seat-feed cells Staging passes --admin-audience for the reserve guard without durable rehome verification, which the parser refused. Only the generation and control origin now travel together, and they still need the audience. The guard decodes the caller from its token and refuses, by name, a revision whose ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT is another account, instead of reading into a 401. --shadow-seat-feed-cells (preserve, all, or a cell list) is a workflow input on both director deploys; preserve keeps the serving value. A contract test parses every workflow's real argv for both deploy scripts. * fix(relay): the flag tool paces a flip back from the pool size the cell reports An image from before the field falls back to the deployed sizes.
This commit is contained in:
Jinwoo Hong
@@ -18,6 +18,17 @@ on:
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
reserve-placement:
|
||||
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, 'off', dry-run, 'on']
|
||||
shadow-seat-feed-cells:
|
||||
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
prune-incompatible-revisions:
|
||||
description: Retain only the newly verified serving and rollback revisions
|
||||
required: true
|
||||
@@ -68,6 +79,8 @@ jobs:
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
|
||||
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
|
||||
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
|
||||
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
|
||||
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
|
||||
# Floor the served revision must keep, matching relay_min_instances in
|
||||
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
|
||||
@@ -228,7 +241,9 @@ jobs:
|
||||
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
|
||||
--release-id "${RELEASE_ID}" \
|
||||
--regional-placement-secret-version "${target_version}" \
|
||||
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}"
|
||||
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" \
|
||||
--reserve-placement "${RESERVE_PLACEMENT}" \
|
||||
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}"
|
||||
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
|
||||
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
|
||||
|
||||
|
||||
@@ -7,6 +7,17 @@ on:
|
||||
description: Exact checked-in production Relay sha256 digest to deploy
|
||||
required: true
|
||||
type: string
|
||||
reserve-placement:
|
||||
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, 'off', dry-run, 'on']
|
||||
shadow-seat-feed-cells:
|
||||
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -87,6 +98,11 @@ jobs:
|
||||
test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}"
|
||||
|
||||
- name: Deploy director blue/green
|
||||
env:
|
||||
# An image change reads every cell's admit mode before the traffic move.
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
|
||||
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
|
||||
run: |
|
||||
regional_version="$(gcloud secrets versions describe latest \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
|
||||
@@ -104,6 +120,9 @@ jobs:
|
||||
--asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \
|
||||
--regional-placement-secret-version "${regional_version}" \
|
||||
--min-instances 0 \
|
||||
--admin-audience https://relay-staging.onorca.dev/v1/admin/drain \
|
||||
--reserve-placement "${RESERVE_PLACEMENT}" \
|
||||
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" \
|
||||
--release-id "${RELEASE_ID}"
|
||||
|
||||
- name: Smoke director health
|
||||
|
||||
@@ -15,18 +15,26 @@ on:
|
||||
description: Exact cell id, e.g. production-gce-c26
|
||||
required: true
|
||||
type: string
|
||||
readiness-local:
|
||||
description: /ready answers from the cell alone (local readiness)
|
||||
break-glass:
|
||||
description: >-
|
||||
Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as
|
||||
BREAK GLASS <cell-id>; records db without the cell's read-back
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
confirm-reserve:
|
||||
description: Required for admitMode=reserve, typed as RESERVE <cell-id>
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
set:
|
||||
description: >-
|
||||
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
|
||||
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
|
||||
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
|
||||
reregisterInFlight
|
||||
required: true
|
||||
default: 'false'
|
||||
type: choice
|
||||
options: ['false', 'true']
|
||||
ticket-check:
|
||||
description: Shadow-check the director's assignment lease on host hellos
|
||||
required: true
|
||||
default: 'off'
|
||||
type: choice
|
||||
options: ['off', 'shadow']
|
||||
type: string
|
||||
expected-generation:
|
||||
description: Exact current object generation from a dry run, or 0 if none exists
|
||||
required: true
|
||||
@@ -55,15 +63,17 @@ jobs:
|
||||
operate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: production
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
READINESS_LOCAL: ${{ inputs.readiness-local }}
|
||||
TICKET_CHECK: ${{ inputs.ticket-check }}
|
||||
FLAG_CHANGES: ${{ inputs.set }}
|
||||
EXPECTED_GENERATION: ${{ inputs.expected-generation }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CONFIRM_RESERVE: ${{ inputs.confirm-reserve }}
|
||||
BREAK_GLASS: ${{ inputs.break-glass }}
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -95,7 +105,9 @@ jobs:
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" \
|
||||
--readiness-local "${READINESS_LOCAL}" --ticket-check "${TICKET_CHECK}" \
|
||||
--set "${FLAG_CHANGES}" \
|
||||
--director-origin "https://relay.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \
|
||||
--break-glass "${BREAK_GLASS}" \
|
||||
--expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
|
||||
|
||||
|
||||
@@ -11,6 +11,11 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET =
|
||||
export const DIRECTOR_REGIONAL_PLACEMENT_ENV =
|
||||
'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
|
||||
export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
|
||||
// Step 5 placement on the directors; only this deploy changes it, so the reserve guard sees it.
|
||||
export const DIRECTOR_RESERVE_PLACEMENT_ENV = 'ORCA_RELAY_RESERVE_PLACEMENT'
|
||||
const RESERVE_PLACEMENT_MODES = ['off', 'dry-run', 'on']
|
||||
// The cells whose feeds a director polls; a reserve cell outside it has no placing director.
|
||||
export const DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV = 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS'
|
||||
export const DIRECTOR_REHOME_IDENTITY_ENV =
|
||||
'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT'
|
||||
export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE'
|
||||
@@ -250,6 +255,12 @@ export function directorDeploymentEnvironment(config) {
|
||||
config['region-correction-cohort-percent'] !== 'preserve') {
|
||||
environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent'])
|
||||
}
|
||||
if (config['reserve-placement'] !== undefined && config['reserve-placement'] !== 'preserve') {
|
||||
environment[DIRECTOR_RESERVE_PLACEMENT_ENV] = config['reserve-placement']
|
||||
}
|
||||
if (config['shadow-seat-feed-cells'] !== undefined && config['shadow-seat-feed-cells'] !== 'preserve') {
|
||||
environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] = shadowSeatFeedCellsSetting(config['shadow-seat-feed-cells'])
|
||||
}
|
||||
const serviceAccount = projectServiceAccount(config, 'capacity-service-account')
|
||||
const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account')
|
||||
const rehomeDirectorServiceAccount = projectServiceAccount(
|
||||
@@ -271,6 +282,16 @@ export function directorDeploymentEnvironment(config) {
|
||||
return environment
|
||||
}
|
||||
|
||||
// `all` or a comma list of cell ids, as the director's config reads it.
|
||||
function shadowSeatFeedCellsSetting(value) {
|
||||
if (value === 'all') return value
|
||||
const cells = value.split(',').map((cell) => cell.trim())
|
||||
if (cells.length > 100 || cells.some((cell) => !/^[a-z][a-z0-9-]{0,39}$/.test(cell))) {
|
||||
throw new Error('--shadow-seat-feed-cells must be preserve, all, or a comma list of cell ids')
|
||||
}
|
||||
return cells.join(',')
|
||||
}
|
||||
|
||||
export function environmentUpdateValue(environment) {
|
||||
const entries = Object.entries(environment)
|
||||
if (entries.every(([key, value]) => !key.includes(',') && !value.includes(','))) {
|
||||
@@ -315,11 +336,19 @@ export function parseArguments(argv) {
|
||||
values['rehome-audience'] !== undefined ||
|
||||
values['expected-rehome-generation'] !== undefined ||
|
||||
values['rehome-control-origin'] !== undefined ||
|
||||
values['region-correction-cohort-percent'] !== undefined
|
||||
values['region-correction-cohort-percent'] !== undefined ||
|
||||
values['reserve-placement'] !== undefined ||
|
||||
values['shadow-seat-feed-cells'] !== undefined
|
||||
) {
|
||||
throw new Error('director configuration arguments require --role director')
|
||||
}
|
||||
}
|
||||
if (
|
||||
values['reserve-placement'] !== undefined &&
|
||||
!['preserve', ...RESERVE_PLACEMENT_MODES].includes(values['reserve-placement'])
|
||||
) {
|
||||
throw new Error('--reserve-placement must be preserve, off, dry-run or on')
|
||||
}
|
||||
if (
|
||||
values.role === 'director' &&
|
||||
values['capacity-cell-id'] !== undefined &&
|
||||
@@ -378,12 +407,14 @@ export function parseArguments(argv) {
|
||||
throw new Error('--rehome-audience must be an exact host-drain HTTPS URL')
|
||||
}
|
||||
}
|
||||
const controlArguments = [
|
||||
'expected-rehome-generation',
|
||||
'rehome-control-origin',
|
||||
'admin-audience'
|
||||
].map((key) => values[key] !== undefined)
|
||||
if (controlArguments.some(Boolean) && !controlArguments.every(Boolean)) {
|
||||
// --admin-audience alone serves the reserve guard's reads; durable verification needs all three.
|
||||
const controlArguments = ['expected-rehome-generation', 'rehome-control-origin'].map(
|
||||
(key) => values[key] !== undefined
|
||||
)
|
||||
if (
|
||||
controlArguments.some(Boolean) &&
|
||||
(!controlArguments.every(Boolean) || values['admin-audience'] === undefined)
|
||||
) {
|
||||
throw new Error('durable rehome verification arguments must be configured together')
|
||||
}
|
||||
if (
|
||||
@@ -429,9 +460,35 @@ export function suppliedAdminIdentityToken(environment = process.env) {
|
||||
return token
|
||||
}
|
||||
|
||||
// The unverified `email` claim, only to name the caller in an error; the director verifies it.
|
||||
export function adminTokenEmail(token) {
|
||||
try {
|
||||
const payload = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString('utf8'))
|
||||
return typeof payload?.email === 'string' ? payload.email : undefined
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
// cell-admit-mode answers only the revision's deploy identity, so a drifted one (a revision that
|
||||
// trusts another account than the workflow's) fails here by name instead of as a bare 401.
|
||||
export function assertDirectorTrustsAdminCaller(token, environment, label) {
|
||||
const trusted = environment.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT
|
||||
const caller = adminTokenEmail(token)
|
||||
if (trusted === undefined || caller === undefined || caller === trusted) return
|
||||
throw new Error(
|
||||
`the ${label} director trusts ${trusted} for deploy admin reads, but this deploy authenticates as ${caller}; ` +
|
||||
'run as that identity or correct the revision\'s ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT'
|
||||
)
|
||||
}
|
||||
|
||||
function adminIdentityToken(config) {
|
||||
const supplied = suppliedAdminIdentityToken()
|
||||
if (supplied === null && !config['admin-audience']) {
|
||||
throw new Error('admin reads need --admin-audience or a supplied ORCA_RELAY_ADMIN_ID_TOKEN')
|
||||
}
|
||||
return (
|
||||
suppliedAdminIdentityToken() ??
|
||||
supplied ??
|
||||
commandText(['auth', 'print-identity-token', `--audiences=${config['admin-audience']}`], {
|
||||
sensitive: true
|
||||
})
|
||||
@@ -721,6 +778,108 @@ export async function assertRegionalRehomeDisabled(
|
||||
return result.control
|
||||
}
|
||||
|
||||
// `undefined` when the director predates the reserve census (its router 404s the route).
|
||||
export async function readReserveCells(config, origin, cellIds, fetchImpl = fetch) {
|
||||
const reserve = []
|
||||
for (const cellId of cellIds) {
|
||||
const response = await fetchImpl(`${origin}/v1/admin/cell-admit-mode`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${adminIdentityToken(config)}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ v: 1, cellId }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
const result = await response.json().catch(() => null)
|
||||
if (response.status === 404 && result === null) return undefined
|
||||
if (!response.ok || result?.v !== 1 || !['db', 'reserve'].includes(result.admitMode)) {
|
||||
throw new Error(`cell admit mode read failed for ${cellId}: ${result?.error ?? response.status}`)
|
||||
}
|
||||
if (result.admitMode === 'reserve') reserve.push(cellId)
|
||||
}
|
||||
return reserve
|
||||
}
|
||||
|
||||
// A director without the census places hosts on reserve cells from stale rows, so traffic may
|
||||
// move between revisions that differ in it only while every cell is db.
|
||||
export function assertReserveCompatibleTrafficMove(servingReserve, candidateReserve) {
|
||||
if ((servingReserve === undefined) === (candidateReserve === undefined)) return
|
||||
const reserve = servingReserve ?? candidateReserve
|
||||
if (reserve.length > 0) {
|
||||
throw new Error(
|
||||
`director revisions differ in reserve-mode support while cells are reserve (${reserve.join(', ')}); flip them to db first`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// With placement off no director books a reserve cell, so each one's dead-man trips (60-120 s)
|
||||
// and every host re-registers at once: the cells go to db first, on purpose.
|
||||
export function assertReservePlacementKept(servingPlacement, candidatePlacement, reserve) {
|
||||
if (servingPlacement !== 'on' || candidatePlacement === 'on') return
|
||||
if (reserve !== undefined && reserve.length > 0) {
|
||||
throw new Error(
|
||||
`reserve placement would turn ${candidatePlacement} while cells are reserve (${reserve.join(', ')}); flip them to db first`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Placement as the director built it (runtime-status), not as its env asks; an image from before
|
||||
// the field reports nothing, and only its env can say.
|
||||
export async function readDirectorPlacement(config, origin, fetchImpl = fetch) {
|
||||
const response = await fetchImpl(`${origin}/v1/admin/runtime-status`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${adminIdentityToken(config)}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ v: 1 }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
const result = await response.json().catch(() => null)
|
||||
if (!response.ok || result?.v !== 1 || result.role !== 'director') {
|
||||
throw new Error(`director runtime status failed at ${origin}: ${result?.error ?? response.status}`)
|
||||
}
|
||||
if (result.reservePlacement === undefined) return 'unreported'
|
||||
if (!RESERVE_PLACEMENT_MODES.includes(result.reservePlacement)) {
|
||||
throw new Error(`director at ${origin} reports reserve placement ${result.reservePlacement}`)
|
||||
}
|
||||
return result.reservePlacement
|
||||
}
|
||||
|
||||
const shadowCellSet = (value) =>
|
||||
(value ?? '').trim() === 'all'
|
||||
? 'all'
|
||||
: new Set(
|
||||
(value ?? '')
|
||||
.split(',')
|
||||
.map((cell) => cell.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
|
||||
// A reserve cell the directors stop polling gets no booking and no reserver poll: its dead-man
|
||||
// trips. Shrinking the shadow-cell list past one is refused until that cell is db.
|
||||
export function assertShadowCellsKeepReserveCells(servingCells, candidateCells, reserve) {
|
||||
if (reserve === undefined || reserve.length === 0) return
|
||||
const before = shadowCellSet(servingCells)
|
||||
const after = shadowCellSet(candidateCells)
|
||||
if (after === 'all') return
|
||||
const dropped = reserve.filter((cell) => (before === 'all' || before.has(cell)) && !after.has(cell))
|
||||
if (dropped.length > 0) {
|
||||
throw new Error(
|
||||
`the shadow seat-feed cells would drop reserve cells (${dropped.join(', ')}); flip them to db first`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function directorCellIds(...environments) {
|
||||
const ids = new Set()
|
||||
for (const environment of environments) {
|
||||
for (const cell of JSON.parse(environment.ORCA_RELAY_CELLS_JSON ?? '[]')) ids.add(cell.id)
|
||||
}
|
||||
return [...ids]
|
||||
}
|
||||
|
||||
function assertDirectorRevisionIdentity(revision, config) {
|
||||
const expectedRuntimeServiceAccount = config['runtime-service-account']
|
||||
if (
|
||||
@@ -741,6 +900,9 @@ export async function deployDirector(config, tag, overrides = {}) {
|
||||
updateTraffic,
|
||||
waitForHealth,
|
||||
assertRegionalRehomeDisabled,
|
||||
readReserveCells,
|
||||
readDirectorPlacement,
|
||||
adminIdentityToken,
|
||||
...overrides
|
||||
}
|
||||
// Why: gcloud does not carry minScale onto a new revision, and the candidate below takes
|
||||
@@ -801,6 +963,12 @@ export async function deployDirector(config, tag, overrides = {}) {
|
||||
deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent(
|
||||
currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0'
|
||||
)
|
||||
if (currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] !== undefined) {
|
||||
deploymentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] ??= currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV]
|
||||
}
|
||||
if (currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] !== undefined) {
|
||||
deploymentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] ??= currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]
|
||||
}
|
||||
if (config['region-correction-cohort-percent'] !== undefined &&
|
||||
config['region-correction-cohort-percent'] !== 'preserve' &&
|
||||
config['expected-rehome-generation'] === undefined) {
|
||||
@@ -911,6 +1079,46 @@ export async function deployDirector(config, tag, overrides = {}) {
|
||||
}
|
||||
await operations.waitForHealth(candidate.origin, requiredCapacityProtocol)
|
||||
await verifyRehomeDisabled(candidate.origin)
|
||||
// The same image and the same placement and shadow-cell settings support the same things;
|
||||
// anything else needs every cell's admit mode (and the admin token that reads it).
|
||||
const sameImage =
|
||||
servingRevision.spec?.containers?.[0]?.image?.split('@').at(-1) === config.image?.split('@').at(-1)
|
||||
const sameSetting = (name) => (currentEnvironment[name] ?? '') === (environment[name] ?? '')
|
||||
if (sameImage && sameSetting(DIRECTOR_RESERVE_PLACEMENT_ENV) && sameSetting(DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV)) {
|
||||
console.warn(JSON.stringify({ event: 'director_reserve_support_checked', skipped: 'same image and settings' }))
|
||||
} else {
|
||||
const servingOrigin = initialService.status?.url
|
||||
if (!servingOrigin) throw new Error('relay service reports no URL for the serving revision')
|
||||
const token = operations.adminIdentityToken(config)
|
||||
assertDirectorTrustsAdminCaller(token, currentEnvironment, 'serving')
|
||||
assertDirectorTrustsAdminCaller(token, environment, 'candidate')
|
||||
const cellIds = directorCellIds(currentEnvironment, environment)
|
||||
const servingReserve = await operations.readReserveCells(config, servingOrigin, cellIds)
|
||||
const candidateReserve = await operations.readReserveCells(config, candidate.origin, cellIds)
|
||||
const reserve = candidateReserve ?? servingReserve
|
||||
// Placement as each director built it, which an env string alone does not say.
|
||||
const placementAt = async (origin, env) => {
|
||||
const placement = await operations.readDirectorPlacement(config, origin)
|
||||
return placement === 'unreported' ? (env[DIRECTOR_RESERVE_PLACEMENT_ENV] ?? 'off') : placement
|
||||
}
|
||||
const servingPlacement = await placementAt(servingOrigin, currentEnvironment)
|
||||
const candidatePlacement = await placementAt(candidate.origin, environment)
|
||||
assertReserveCompatibleTrafficMove(servingReserve, candidateReserve)
|
||||
assertReservePlacementKept(servingPlacement, candidatePlacement, reserve)
|
||||
assertShadowCellsKeepReserveCells(
|
||||
currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV],
|
||||
environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV],
|
||||
reserve
|
||||
)
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'director_reserve_support_checked',
|
||||
reservePlacement: { serving: servingPlacement, candidate: candidatePlacement },
|
||||
serving: servingReserve === undefined ? 'unsupported' : servingReserve,
|
||||
candidate: candidateReserve === undefined ? 'unsupported' : candidateReserve
|
||||
})
|
||||
)
|
||||
}
|
||||
operations.updateTraffic(config, [`--to-tags=${tag}=100`])
|
||||
promoted = true
|
||||
removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG]))
|
||||
|
||||
@@ -13,13 +13,19 @@ import {
|
||||
DIRECTOR_REHOME_AUDIENCE_ENV,
|
||||
DIRECTOR_REHOME_IDENTITY_ENV,
|
||||
assertRegionalRehomeDisabled,
|
||||
assertReserveCompatibleTrafficMove,
|
||||
assertShadowCellsKeepReserveCells,
|
||||
deployDirector,
|
||||
DIRECTOR_RESERVE_PLACEMENT_ENV,
|
||||
directorDeploymentEnvironment,
|
||||
DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV,
|
||||
directorCellSetAddition,
|
||||
directorStartupProbeArguments,
|
||||
directorTopologyChange,
|
||||
environmentUpdateValue,
|
||||
parseArguments,
|
||||
readDirectorPlacement,
|
||||
readReserveCells,
|
||||
revisionEnvironment,
|
||||
revisionSecretEnvironment,
|
||||
suppliedAdminIdentityToken,
|
||||
@@ -418,6 +424,9 @@ test('requires exactly one active revision and reads queried tag metadata', () =
|
||||
assert.deepEqual(trafficTags(service), ['candidate-123'])
|
||||
})
|
||||
|
||||
const adminToken = (email) =>
|
||||
`e30.${Buffer.from(JSON.stringify(email ? { email } : {})).toString('base64url')}.sig`
|
||||
|
||||
function directorHarness({
|
||||
deployFailure,
|
||||
deleteFailure,
|
||||
@@ -428,7 +437,11 @@ function directorHarness({
|
||||
// Reproduces gcloud dropping minScale from a newly created revision.
|
||||
dropRequestedMinimum = false,
|
||||
servingServiceAccount,
|
||||
servingImageDigest = `sha256:${'f'.repeat(64)}`
|
||||
servingImageDigest = `sha256:${'f'.repeat(64)}`,
|
||||
reserveCells = {},
|
||||
servingEnvironment = {},
|
||||
placements = {},
|
||||
callerEmail
|
||||
} = {}) {
|
||||
const state = {
|
||||
activeRevision: 'relay-00001-old',
|
||||
@@ -439,7 +452,8 @@ function directorHarness({
|
||||
'relay-00001-old',
|
||||
{
|
||||
env: {
|
||||
ORCA_RELAY_ROLE: 'director'
|
||||
ORCA_RELAY_ROLE: 'director',
|
||||
...servingEnvironment
|
||||
},
|
||||
secrets: {
|
||||
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
||||
@@ -462,6 +476,7 @@ function directorHarness({
|
||||
const operations = {
|
||||
describeService: () => ({
|
||||
status: {
|
||||
url: 'https://relay-hash-uc.a.run.app',
|
||||
traffic: [
|
||||
{ percent: 100, revisionName: state.activeRevision },
|
||||
...[...state.tags].map(([tag, revisionName]) => ({
|
||||
@@ -501,7 +516,8 @@ function directorHarness({
|
||||
deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => {
|
||||
const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new`
|
||||
state.revisions.set(revision, {
|
||||
env: { ORCA_RELAY_ROLE: 'director', ...env },
|
||||
// gcloud --update-env-vars keeps every variable the deploy does not name.
|
||||
env: { ORCA_RELAY_ROLE: 'director', ...servingEnvironment, ...env },
|
||||
secrets: {
|
||||
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
||||
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
||||
@@ -544,7 +560,10 @@ function directorHarness({
|
||||
},
|
||||
waitForHealth: async (_origin, connectionCapacityProtocol) => {
|
||||
healthProtocols.push(connectionCapacityProtocol)
|
||||
}
|
||||
},
|
||||
readReserveCells: async (_config, origin) => reserveCells[origin],
|
||||
readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported',
|
||||
adminIdentityToken: () => adminToken(callerEmail)
|
||||
}
|
||||
return { state, removed, healthProtocols, operations }
|
||||
}
|
||||
@@ -854,3 +873,206 @@ test('sets a reviewed cohort only behind repeated disabled-control verification'
|
||||
assert.ok(verified >= 2)
|
||||
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1')
|
||||
})
|
||||
|
||||
test('director deploy refuses a traffic move between reserve-support revisions while a cell is reserve', async () => {
|
||||
const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app'
|
||||
const oldServing = directorHarness({
|
||||
reserveCells: { [candidateOrigin]: ['production-gce-c3'] }
|
||||
})
|
||||
await assert.rejects(
|
||||
deployDirector({}, 'candidate-new', oldServing.operations),
|
||||
/differ in reserve-mode support while cells are reserve \(production-gce-c3\)/
|
||||
)
|
||||
assert.equal(oldServing.state.activeRevision, 'relay-00001-old')
|
||||
const rollbackToOld = directorHarness({
|
||||
reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'] }
|
||||
})
|
||||
await assert.rejects(deployDirector({}, 'candidate-new', rollbackToOld.operations), /flip them to db first/)
|
||||
assert.equal(rollbackToOld.state.activeRevision, 'relay-00001-old')
|
||||
const allDb = directorHarness({ reserveCells: { [candidateOrigin]: [] } })
|
||||
await deployDirector({}, 'candidate-new', allDb.operations)
|
||||
assert.equal(allDb.state.activeRevision, 'relay-00003-new')
|
||||
})
|
||||
|
||||
test('two revisions that both read the census may move traffic with reserve cells', () => {
|
||||
assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(['c3'], ['c3']))
|
||||
assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(undefined, undefined))
|
||||
assert.throws(() => assertReserveCompatibleTrafficMove(undefined, ['c3']), /flip them to db first/)
|
||||
})
|
||||
|
||||
test('reads a census-less director as unsupported and fails closed on any other refusal', async () => {
|
||||
const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc'
|
||||
const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' }
|
||||
try {
|
||||
assert.equal(
|
||||
await readReserveCells(config, 'https://old.example.test', ['c1'], async () =>
|
||||
new Response('404 Not Found', { status: 404 })
|
||||
),
|
||||
undefined
|
||||
)
|
||||
const modes = { c1: 'db', c2: 'reserve' }
|
||||
assert.deepEqual(
|
||||
await readReserveCells(config, 'https://new.example.test', ['c1', 'c2'], async (url, init) => {
|
||||
assert.equal(url, 'https://new.example.test/v1/admin/cell-admit-mode')
|
||||
const { cellId } = JSON.parse(init.body)
|
||||
return new Response(JSON.stringify({ v: 1, cellId, admitMode: modes[cellId], updatedAt: 1 }))
|
||||
}),
|
||||
['c2']
|
||||
)
|
||||
await assert.rejects(
|
||||
readReserveCells(config, 'https://new.example.test', ['c1'], async () =>
|
||||
new Response(JSON.stringify({ error: 'invalid_token' }), { status: 401 })
|
||||
),
|
||||
/cell admit mode read failed for c1: invalid_token/
|
||||
)
|
||||
} finally {
|
||||
if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment
|
||||
}
|
||||
})
|
||||
|
||||
test('a same-image deploy reads no admit modes, and keeps the serving reserve placement', async () => {
|
||||
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
||||
const harness = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' } })
|
||||
let reads = 0
|
||||
harness.operations.readReserveCells = async () => {
|
||||
reads += 1
|
||||
return []
|
||||
}
|
||||
await deployDirector({ image: `relay@${servingImageDigest}` }, 'candidate-new', harness.operations)
|
||||
assert.equal(reads, 0)
|
||||
assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'on')
|
||||
})
|
||||
|
||||
test('turning reserve placement off is refused while any cell is reserve, even on the same image', async () => {
|
||||
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
||||
const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app'
|
||||
const reserved = directorHarness({
|
||||
servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' },
|
||||
reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] },
|
||||
placements: { 'https://relay-hash-uc.a.run.app': 'on', [candidateOrigin]: 'off' }
|
||||
})
|
||||
await assert.rejects(
|
||||
deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', reserved.operations),
|
||||
/reserve placement would turn off while cells are reserve \(production-gce-c3\); flip them to db first/
|
||||
)
|
||||
assert.equal(reserved.state.activeRevision, 'relay-00001-old')
|
||||
const allDb = directorHarness({
|
||||
servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' },
|
||||
reserveCells: { 'https://relay-hash-uc.a.run.app': [], [candidateOrigin]: [] }
|
||||
})
|
||||
await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', allDb.operations)
|
||||
assert.equal(allDb.state.revisions.get(allDb.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off')
|
||||
})
|
||||
|
||||
test('--reserve-placement takes only a known mode, and only for a director', () => {
|
||||
const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'a'.repeat(64)}`, '--release-id', '1']
|
||||
assert.equal(parseArguments([...base, '--role', 'director', '--reserve-placement', 'on'])['reserve-placement'], 'on')
|
||||
assert.throws(() => parseArguments([...base, '--role', 'director', '--reserve-placement', 'yes']), /must be preserve, off, dry-run or on/)
|
||||
assert.throws(
|
||||
() => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--reserve-placement', 'on']),
|
||||
/require --role director/
|
||||
)
|
||||
assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'preserve' })[DIRECTOR_RESERVE_PLACEMENT_ENV], undefined)
|
||||
assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'dry-run' })[DIRECTOR_RESERVE_PLACEMENT_ENV], 'dry-run')
|
||||
})
|
||||
|
||||
test('the guard reads placement as built, so a director that never placed does not block turning it off', async () => {
|
||||
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
||||
const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app'
|
||||
// Configured on, but the serving director reports off: it was never feeding the dead-man.
|
||||
const harness = directorHarness({
|
||||
servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' },
|
||||
reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] },
|
||||
placements: { 'https://relay-hash-uc.a.run.app': 'off', [candidateOrigin]: 'off' }
|
||||
})
|
||||
await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations)
|
||||
assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off')
|
||||
})
|
||||
|
||||
test('a shadow-cell change on the same image is checked, and a shrink past a reserve cell is refused', () => {
|
||||
assert.throws(
|
||||
() => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', ['production-gce-c3']),
|
||||
/would drop reserve cells \(production-gce-c3\); flip them to db first/
|
||||
)
|
||||
assert.throws(() => assertShadowCellsKeepReserveCells('all', 'production-gce-c1', ['production-gce-c3']), /production-gce-c3/)
|
||||
assert.throws(() => assertShadowCellsKeepReserveCells('all', undefined, ['production-gce-c3']), /production-gce-c3/)
|
||||
assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'all', ['production-gce-c3']))
|
||||
assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', []))
|
||||
// A reserve cell the serving list never polled is not being dropped by this deploy.
|
||||
assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'production-gce-c1', ['production-gce-c3']))
|
||||
})
|
||||
|
||||
test('reads a director placement from runtime-status, and reports an image without the field', async () => {
|
||||
const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc'
|
||||
const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' }
|
||||
try {
|
||||
const answer = (body) => async () => new Response(JSON.stringify({ v: 1, role: 'director', ...body }))
|
||||
assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'on' })), 'on')
|
||||
assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({})), 'unreported')
|
||||
await assert.rejects(readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'maybe' })), /reports reserve placement maybe/)
|
||||
await assert.rejects(
|
||||
readDirectorPlacement(config, 'https://d.example.test', async () => new Response('{}', { status: 401 })),
|
||||
/runtime status failed/
|
||||
)
|
||||
} finally {
|
||||
if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment
|
||||
}
|
||||
})
|
||||
|
||||
// Staging's director trusted gha-deploy while the workflow authenticated as gha-relay: a bare 401.
|
||||
test('the guard names the deploy identity a drifted director trusts instead of reading into a 401', async () => {
|
||||
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
||||
const harness = directorHarness({
|
||||
servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-deploy@p.iam.gserviceaccount.com' },
|
||||
callerEmail: 'gha-relay@p.iam.gserviceaccount.com'
|
||||
})
|
||||
await assert.rejects(
|
||||
deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations),
|
||||
/serving director trusts gha-deploy@p\.iam\.gserviceaccount\.com for deploy admin reads, but this deploy authenticates as gha-relay@p\.iam\.gserviceaccount\.com/
|
||||
)
|
||||
const trusted = directorHarness({
|
||||
servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-relay@p.iam.gserviceaccount.com' },
|
||||
callerEmail: 'gha-relay@p.iam.gserviceaccount.com'
|
||||
})
|
||||
await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', trusted.operations)
|
||||
})
|
||||
|
||||
test('a director deploy takes --admin-audience alone for the reserve guard', () => {
|
||||
const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'e'.repeat(64)}`, '--role', 'director', '--release-id', 'x']
|
||||
const audience = ['--admin-audience', 'https://relay.example.test']
|
||||
assert.equal(parseArguments([...base, ...audience])['admin-audience'], 'https://relay.example.test')
|
||||
assert.throws(
|
||||
() => parseArguments([...base, '--expected-rehome-generation', '1', '--rehome-control-origin', 'https://relay.example.test']),
|
||||
/durable rehome verification arguments must be configured together/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseArguments([...base, ...audience, '--expected-rehome-generation', '1']),
|
||||
/durable rehome verification arguments must be configured together/
|
||||
)
|
||||
})
|
||||
|
||||
test('--shadow-seat-feed-cells sets the director setting, and preserve keeps the serving one', async () => {
|
||||
assert.equal(
|
||||
directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'c1, c2' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV],
|
||||
'c1,c2'
|
||||
)
|
||||
assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'all' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'all')
|
||||
assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'preserve' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], undefined)
|
||||
assert.throws(() => directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'C1;x' }), /must be preserve, all, or a comma list/)
|
||||
const servingImageDigest = `sha256:${'f'.repeat(64)}`
|
||||
const harness = directorHarness({ servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' } })
|
||||
await deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'preserve' }, 'candidate-new', harness.operations)
|
||||
assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'c1,c3')
|
||||
const shrink = directorHarness({
|
||||
servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' },
|
||||
reserveCells: { 'https://relay-hash-uc.a.run.app': ['c3'], 'https://candidate-new---relay-hash-uc.a.run.app': ['c3'] }
|
||||
})
|
||||
await assert.rejects(
|
||||
deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'c1' }, 'candidate-new', shrink.operations),
|
||||
/would drop reserve cells \(c3\)/
|
||||
)
|
||||
})
|
||||
|
||||
@@ -16,18 +16,144 @@ export const READ_BACK_TIMEOUT_MS = 15_000
|
||||
const READ_BACK_INTERVAL_MS = 1_000
|
||||
const STORAGE_ROOT = 'https://storage.googleapis.com'
|
||||
|
||||
// Every switch a cell image may know, with what an absent key means on the cell. A cell
|
||||
// resets a key missing from its object to this default, so a write must carry every key it
|
||||
// does not mean to change: the tool reads the current object and changes only what is named.
|
||||
const enumOf = (...options) => (value) => (options.includes(value) ? value : undefined)
|
||||
const booleanValue = (value) => (value === 'true' ? true : value === 'false' ? false : undefined)
|
||||
const numberIn = (min, max, integer) => (value) => {
|
||||
const number = Number(value)
|
||||
if (value === '' || !Number.isFinite(number) || number < min || number > max) return undefined
|
||||
if (integer && !Number.isInteger(number)) return undefined
|
||||
return number
|
||||
}
|
||||
export const CELL_FLAG_SPECS = {
|
||||
readinessLocal: { parse: booleanValue, default: false },
|
||||
ticketCheck: { parse: enumOf('off', 'shadow', 'enforce'), default: 'off' },
|
||||
admitMode: { parse: enumOf('db', 'reserve'), default: 'db' },
|
||||
// Absent means the cell's region default.
|
||||
intakePerSec: {
|
||||
parse: (value) => {
|
||||
const rate = numberIn(0, 1_000, false)(value)
|
||||
return rate === 0 ? undefined : rate
|
||||
}
|
||||
},
|
||||
reserveDryRun: { parse: booleanValue, default: false },
|
||||
rejectionFence: { parse: booleanValue, default: true },
|
||||
// Absent keeps the cell's boot value.
|
||||
readTimeoutMarginMs: { parse: numberIn(1_000, 60_000, true) },
|
||||
// Flip-back leases in flight; absent is a third of the cell's pool.
|
||||
reregisterInFlight: { parse: numberIn(1, 16, true) }
|
||||
}
|
||||
|
||||
export const RESERVE_CONFIRMATION = 'RESERVE'
|
||||
export const BREAK_GLASS_CONFIRMATION = 'BREAK GLASS'
|
||||
// Longer than a director's cached read of the admit-mode table (5 s).
|
||||
export const RESERVE_MODE_CACHE_WAIT_MS = 10_000
|
||||
// How long a flip back may take to lease every control the cell admitted from memory: its
|
||||
// control count at a per-lane pace below the measured 0.38/s (170 ms trips), within the job.
|
||||
export const REREGISTER_WAIT_MIN_MS = 8 * 60_000
|
||||
export const REREGISTER_WAIT_MAX_MS = 50 * 60_000
|
||||
const REREGISTER_CONTROLS_PER_LANE_PER_SEC = 0.3
|
||||
// The cell takes a third of its database pool, and caps the switch at the pool less two
|
||||
// (host-session-registry.ts reregisterInFlightLimit). An image from before runtime-status
|
||||
// reported the pool falls back to the deployed sizes (US 10, Asia 16).
|
||||
const CELL_DATABASE_POOL_MAX = { 'asia-east2': 16 }
|
||||
const CELL_DATABASE_POOL_MAX_DEFAULT = 10
|
||||
|
||||
export function reregisterLanes(runtime) {
|
||||
const reported = runtime?.databasePoolMax
|
||||
const pool =
|
||||
Number.isInteger(reported) && reported > 0
|
||||
? reported
|
||||
: (CELL_DATABASE_POOL_MAX[runtime?.region] ?? CELL_DATABASE_POOL_MAX_DEFAULT)
|
||||
const flag = runtime?.flagsApplied?.flags?.reregisterInFlight
|
||||
if (flag === undefined) return Math.max(1, Math.floor(pool / 3))
|
||||
return Math.max(1, Math.min(Number(flag), pool - 2))
|
||||
}
|
||||
|
||||
export function reregisterWaitMs(runtime) {
|
||||
const controls = Number(runtime?.runtime?.controls ?? 0)
|
||||
const lanes = reregisterLanes(runtime)
|
||||
const estimate = (controls / (lanes * REREGISTER_CONTROLS_PER_LANE_PER_SEC)) * 1_000
|
||||
return Math.min(REREGISTER_WAIT_MAX_MS, Math.max(REREGISTER_WAIT_MIN_MS, Math.ceil(estimate)))
|
||||
}
|
||||
|
||||
// An image from before `supportedFlags` accepts the keys its applied flags list, and only
|
||||
// ticketCheck off/shadow: anything else is dropped there, or voids the whole object.
|
||||
function legacySupportedFlags(appliedFlags) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(appliedFlags ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
key === 'ticketCheck'
|
||||
? { type: 'enum', values: ['off', 'shadow'] }
|
||||
: { type: typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'number' : 'enum', values: [value] }
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
function accepts(spec, value) {
|
||||
if (spec?.type === 'boolean') return typeof value === 'boolean'
|
||||
if (spec?.type === 'enum') return spec.values.includes(value)
|
||||
if (spec?.type === 'number') {
|
||||
return (
|
||||
typeof value === 'number' &&
|
||||
(spec.min === undefined || value >= spec.min) &&
|
||||
(spec.max === undefined || value <= spec.max) &&
|
||||
(!spec.integer || Number.isInteger(value))
|
||||
)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The whole object as it will be written, not only the change: a bad key or value already in
|
||||
// the file would void or be dropped on every later write too.
|
||||
export function assertSupportedObject(flags, runtime) {
|
||||
const supported = runtime.supportedFlags ?? legacySupportedFlags(runtime.flagsApplied?.flags)
|
||||
for (const [key, value] of Object.entries(flags)) {
|
||||
if (!accepts(supported[key], value)) {
|
||||
throw new Error(
|
||||
`the cell's image does not support ${key}=${JSON.stringify(value)}; roll the image first`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `key=value,key=value`; `key=default` removes the key so the cell uses its default.
|
||||
export function parseFlagChanges(text) {
|
||||
const changes = {}
|
||||
for (const part of (text ?? '').split(',').map((entry) => entry.trim()).filter(Boolean)) {
|
||||
const separator = part.indexOf('=')
|
||||
const key = separator < 0 ? part : part.slice(0, separator)
|
||||
const raw = separator < 0 ? '' : part.slice(separator + 1).trim()
|
||||
const spec = CELL_FLAG_SPECS[key]
|
||||
if (!spec) throw new Error(`unknown switch ${key}`)
|
||||
if (key in changes) throw new Error(`switch ${key} named twice`)
|
||||
if (raw === 'default') {
|
||||
changes[key] = null
|
||||
continue
|
||||
}
|
||||
const value = spec.parse(raw)
|
||||
if (value === undefined) throw new Error(`switch ${key} cannot be ${JSON.stringify(raw)}`)
|
||||
changes[key] = value
|
||||
}
|
||||
if (Object.keys(changes).length === 0) throw new Error('set names no switch to change')
|
||||
return changes
|
||||
}
|
||||
|
||||
// The switches as the cell will apply them: every known key, absent ones at their default.
|
||||
function effectiveFlags(flags) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(CELL_FLAG_SPECS).map(([key, spec]) => [key, flags?.[key] ?? spec.default])
|
||||
)
|
||||
}
|
||||
|
||||
export function parseCellFlagsRequest(values) {
|
||||
const mode = values.mode
|
||||
if (mode !== 'dry-run' && mode !== 'write') throw new Error('mode must be dry-run or write')
|
||||
const cellId = values['cell-id'] ?? ''
|
||||
if (!CELL_ID_PATTERN.test(cellId)) throw new Error('cell id is not a relay GCE cell id')
|
||||
if (values['readiness-local'] !== 'true' && values['readiness-local'] !== 'false') {
|
||||
throw new Error('readiness-local must be true or false')
|
||||
}
|
||||
const ticketCheck = values['ticket-check']
|
||||
if (ticketCheck !== 'off' && ticketCheck !== 'shadow') {
|
||||
throw new Error('ticket-check must be off or shadow')
|
||||
}
|
||||
const changes = parseFlagChanges(values.set)
|
||||
// 0 means "no object yet": the write then creates it and refuses if one appeared.
|
||||
const expectedGeneration = values['expected-generation'] ?? ''
|
||||
if (!/^(0|[1-9][0-9]{0,18})$/.test(expectedGeneration)) {
|
||||
@@ -36,6 +162,29 @@ export function parseCellFlagsRequest(values) {
|
||||
if (mode === 'write' && values.confirmation !== `${WRITE_CONFIRMATION} ${cellId}`) {
|
||||
throw new Error(`write requires the confirmation "${WRITE_CONFIRMATION} ${cellId}"`)
|
||||
}
|
||||
// Reserve moves this cell's hosts off the database: it is typed for this cell, every time.
|
||||
if (
|
||||
mode === 'write' &&
|
||||
changes.admitMode === 'reserve' &&
|
||||
values['confirm-reserve'] !== `${RESERVE_CONFIRMATION} ${cellId}`
|
||||
) {
|
||||
throw new Error(`admitMode=reserve requires --confirm-reserve "${RESERVE_CONFIRMATION} ${cellId}"`)
|
||||
}
|
||||
// Break glass: a dead or unreachable reserve cell cannot report db, so nothing would ever
|
||||
// let the sweeps re-place its hosts. Only admitMode=db, typed for this cell.
|
||||
const breakGlass = values['break-glass'] ?? ''
|
||||
if (breakGlass !== '') {
|
||||
if (breakGlass !== `${BREAK_GLASS_CONFIRMATION} ${cellId}`) {
|
||||
throw new Error(`break glass requires --break-glass "${BREAK_GLASS_CONFIRMATION} ${cellId}"`)
|
||||
}
|
||||
if (mode !== 'write' || changes.admitMode !== null || Object.keys(changes).length !== 1) {
|
||||
throw new Error('break glass only writes admitMode=default (db), on its own')
|
||||
}
|
||||
}
|
||||
const directorOrigin = values['director-origin'] ?? ''
|
||||
if (changes.admitMode !== undefined && !/^https:\/\/relay(-staging)?\.onorca\.dev$/.test(directorOrigin)) {
|
||||
throw new Error('an admitMode change needs --director-origin, where Postgres records it')
|
||||
}
|
||||
const projectId = values['project-id'] ?? ''
|
||||
if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId)) throw new Error('project id is invalid')
|
||||
const cellOrigin = values['cell-origin'] ?? ''
|
||||
@@ -49,16 +198,27 @@ export function parseCellFlagsRequest(values) {
|
||||
mode,
|
||||
cellId,
|
||||
cellOrigin,
|
||||
directorOrigin,
|
||||
breakGlass: breakGlass !== '',
|
||||
projectId,
|
||||
expectedGeneration,
|
||||
object: {
|
||||
v: 1,
|
||||
cellId,
|
||||
flags: { readinessLocal: values['readiness-local'] === 'true', ticketCheck }
|
||||
}
|
||||
changes
|
||||
}
|
||||
}
|
||||
|
||||
// Read-modify-write: the current object's switches with only the named ones changed.
|
||||
export function desiredObject(request, current) {
|
||||
if (current !== null && (current?.v !== 1 || current?.cellId !== request.cellId)) {
|
||||
throw new Error('the current object is not this cell\'s v1 switch object; fix it by hand')
|
||||
}
|
||||
const flags = { ...(current?.flags ?? {}) }
|
||||
for (const [key, value] of Object.entries(request.changes)) {
|
||||
if (value === null) delete flags[key]
|
||||
else flags[key] = value
|
||||
}
|
||||
return { v: 1, cellId: request.cellId, flags }
|
||||
}
|
||||
|
||||
function objectUrl(request, suffix = '') {
|
||||
const bucket = `${request.projectId}-relay-control`
|
||||
return `${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${encodeURIComponent(
|
||||
@@ -101,7 +261,7 @@ async function readCurrentObject(fetchImpl, request, accessToken) {
|
||||
return { generation: String(generation), object: await media.json().catch(() => null) }
|
||||
}
|
||||
|
||||
async function writeObject(fetchImpl, request, accessToken, audit) {
|
||||
async function writeObject(fetchImpl, request, object, accessToken, audit) {
|
||||
const boundary = `relay-cell-flags-${Date.now()}`
|
||||
const metadata = {
|
||||
name: `cells/${request.cellId}.json`,
|
||||
@@ -118,7 +278,7 @@ async function writeObject(fetchImpl, request, accessToken, audit) {
|
||||
`--${boundary}`,
|
||||
'Content-Type: application/json',
|
||||
'',
|
||||
JSON.stringify(request.object),
|
||||
JSON.stringify(object),
|
||||
`--${boundary}--`,
|
||||
''
|
||||
].join('\r\n')
|
||||
@@ -134,21 +294,119 @@ async function writeObject(fetchImpl, request, accessToken, audit) {
|
||||
body
|
||||
}
|
||||
)
|
||||
// A 4xx means nothing was written; a 5xx or a lost reply may have been.
|
||||
if (response.status === 412) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error('the object changed since expected-generation; re-read and retry')
|
||||
throw Object.assign(new Error('the object changed since expected-generation; re-read and retry'), {
|
||||
notWritten: true
|
||||
})
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw Object.assign(new Error(`object write returned ${response.status}`), {
|
||||
notWritten: response.status < 500
|
||||
})
|
||||
}
|
||||
if (!response.ok) throw new Error(`object write returned ${response.status}`)
|
||||
return String((await response.json()).generation)
|
||||
}
|
||||
|
||||
const sameFlags = (left, right) =>
|
||||
left?.readinessLocal === right.readinessLocal && left?.ticketCheck === right.ticketCheck
|
||||
// The census's record (relay_cell_admit_modes, through a director). Without `admitMode` it only
|
||||
// reads. Read back on every write: sweeps act on this, not on the cell's switch file.
|
||||
async function directorAdmitMode(fetchImpl, request, idToken, wait, admitMode) {
|
||||
const response = await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
`${request.directorOrigin}/v1/admin/cell-admit-mode`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1, cellId: request.cellId, ...(admitMode ? { admitMode } : {}) })
|
||||
},
|
||||
{ wait, timeoutMs: 10_000, retryDelayMs: 1_000 }
|
||||
)
|
||||
const body = await response.json().catch(() => null)
|
||||
if (!response.ok) throw new Error(`director refused admitMode: ${response.status} ${body?.error ?? ''}`)
|
||||
if (admitMode && body?.admitMode !== admitMode) {
|
||||
throw new Error(`director recorded admitMode ${body?.admitMode}, not ${admitMode}`)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// Every known key, not just the ones this run changed: a cell that kept an old value of any
|
||||
// switch has not applied this object. A key with no fixed default (the cell reports its own
|
||||
// value) is compared only when the object names it.
|
||||
export function sameFlags(applied, desired) {
|
||||
const expected = effectiveFlags(desired)
|
||||
const actual = effectiveFlags(applied)
|
||||
return Object.entries(CELL_FLAG_SPECS).every(
|
||||
([key, spec]) =>
|
||||
(!('default' in spec) && desired?.[key] === undefined) || actual[key] === expected[key]
|
||||
)
|
||||
}
|
||||
|
||||
// Sweeps resume only once every control the cell admitted from memory holds a lease again.
|
||||
async function waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log }) {
|
||||
let waitMs
|
||||
let deadline = now() + REREGISTER_WAIT_MIN_MS
|
||||
while (now() < deadline) {
|
||||
const runtime = await readRuntime(fetchImpl, request, idToken, sleep).catch(() => null)
|
||||
// An image without the field never admitted from memory.
|
||||
if (runtime && (runtime.admitModeEffective ?? 'db') === 'db') return
|
||||
if (runtime && waitMs === undefined) {
|
||||
waitMs = reregisterWaitMs(runtime)
|
||||
deadline = now() + waitMs
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_reregistration_wait', cellId: request.cellId, waitMs }))
|
||||
}
|
||||
await sleep(5_000)
|
||||
}
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_reregistration_still_running', cellId: request.cellId }))
|
||||
throw new Error(
|
||||
`cell still re-registering after ${waitMs ?? REREGISTER_WAIT_MIN_MS} ms. Its connections are safe and Postgres ` +
|
||||
'keeps reserve, so sweeps stay off the cell. Run again with --set admitMode=db, which records db once ' +
|
||||
'runtime-status reports admitModeEffective=db; to speed it up, name both in one write: ' +
|
||||
'--set admitMode=db,reregisterInFlight=N'
|
||||
)
|
||||
}
|
||||
|
||||
// No cell read-back and no wait for its controls: the cell is not answering. The file says db
|
||||
// so a cell that comes back boots in db, and Postgres says db so the sweeps re-place its hosts.
|
||||
async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep }) {
|
||||
const { accessToken, idToken, audit } = dependencies
|
||||
// Only for a dead cell: one that answers, or still heartbeats, takes the ordinary flip back.
|
||||
const answered = await fetchImpl(`${request.cellOrigin}/v1/admin/runtime-status`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1 }),
|
||||
signal: AbortSignal.timeout(10_000)
|
||||
})
|
||||
.then(async (response) => {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
return response.ok
|
||||
})
|
||||
.catch(() => false)
|
||||
if (answered) throw new Error('the cell answers runtime-status: flip it back with --set admitMode=db, not break glass')
|
||||
const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep)
|
||||
if (recorded?.heartbeatFresh !== false) {
|
||||
throw new Error(
|
||||
`the director ${recorded?.heartbeatFresh ? 'still sees heartbeats from' : 'cannot say whether it hears'} ` +
|
||||
'the cell: break glass is only for a cell that has stopped'
|
||||
)
|
||||
}
|
||||
const current = await readCurrentObject(fetchImpl, request, accessToken)
|
||||
if (current.generation !== request.expectedGeneration) {
|
||||
throw new Error(`current generation is ${current.generation}, not expected ${request.expectedGeneration}`)
|
||||
}
|
||||
const object = desiredObject(request, current.object)
|
||||
const generation = await writeObject(fetchImpl, request, object, accessToken, audit)
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
const result = { event: 'orca_relay_cell_admit_mode_break_glass', cellId: request.cellId, generation, audit }
|
||||
log(JSON.stringify(result))
|
||||
return { ...result, written: true }
|
||||
}
|
||||
|
||||
export async function operateCellFlags(request, dependencies) {
|
||||
const { fetchImpl = fetch, accessToken, idToken, audit, log = console.log } = dependencies
|
||||
const now = dependencies.now ?? Date.now
|
||||
const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
if (request.breakGlass) return await breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep })
|
||||
const runtime = await readRuntime(fetchImpl, request, idToken, sleep)
|
||||
// Proves the origin is this cell, and that its image reads switch files at all.
|
||||
if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) {
|
||||
@@ -156,6 +414,20 @@ export async function operateCellFlags(request, dependencies) {
|
||||
}
|
||||
if (!runtime.flagsApplied) throw new Error('cell image has no flag channel (no flagsApplied)')
|
||||
const current = await readCurrentObject(fetchImpl, request, accessToken)
|
||||
const object = desiredObject(request, current.object)
|
||||
assertSupportedObject(object.flags, runtime)
|
||||
// A tripped dead-man: any new generation that keeps reserve re-arms it, so the cell trips
|
||||
// again and its re-registration restarts. Only a write that also says db is safe.
|
||||
if (
|
||||
runtime.flagsApplied.flags?.admitMode === 'reserve' &&
|
||||
runtime.admitModeEffective === 'db' &&
|
||||
(object.flags.admitMode ?? 'db') === 'reserve'
|
||||
) {
|
||||
throw new Error(
|
||||
'the cell tripped its dead-man (reserve configured, db effective); a write that keeps reserve re-arms it. ' +
|
||||
'Name admitMode=db in the same write, e.g. --set admitMode=db,reregisterInFlight=N'
|
||||
)
|
||||
}
|
||||
const plan = {
|
||||
event: 'orca_relay_cell_flags_plan',
|
||||
mode: request.mode,
|
||||
@@ -163,7 +435,8 @@ export async function operateCellFlags(request, dependencies) {
|
||||
currentGeneration: current.generation,
|
||||
currentObject: current.object,
|
||||
appliedBefore: runtime.flagsApplied,
|
||||
desired: request.object
|
||||
changes: request.changes,
|
||||
desired: object
|
||||
}
|
||||
// Before the generation check, so a dry run always shows the generation to pass next.
|
||||
log(JSON.stringify(plan))
|
||||
@@ -172,8 +445,53 @@ export async function operateCellFlags(request, dependencies) {
|
||||
`current generation is ${current.generation}, not expected ${request.expectedGeneration}`
|
||||
)
|
||||
}
|
||||
// The cell applies every readable write; a generation it did not apply was voided there. A
|
||||
// write of an object this image accepts whole (checked above) repairs it.
|
||||
if (current.generation !== '0' && String(runtime.flagsApplied.generation) !== current.generation) {
|
||||
log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_cell_flags_repairing_unapplied_object',
|
||||
cellId: request.cellId,
|
||||
appliedGeneration: runtime.flagsApplied.generation,
|
||||
currentGeneration: current.generation
|
||||
})
|
||||
)
|
||||
}
|
||||
const admitModeChange =
|
||||
request.changes.admitMode === undefined ? undefined : (request.changes.admitMode ?? 'db')
|
||||
if (admitModeChange !== undefined) {
|
||||
const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep)
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded_before', ...recorded }))
|
||||
}
|
||||
if (request.mode === 'dry-run') return { ...plan, written: false }
|
||||
const generation = await writeObject(fetchImpl, request, accessToken, audit)
|
||||
// Sweeps skip the cell before it starts admitting from memory, never after.
|
||||
let generation
|
||||
if (admitModeChange === 'reserve') {
|
||||
const before = await directorAdmitMode(fetchImpl, request, idToken, sleep)
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve')
|
||||
// Every director's cached set expires (5 s) before the cell starts admitting from memory;
|
||||
// the second write re-runs the open-flow check over anything started meanwhile.
|
||||
await sleep(RESERVE_MODE_CACHE_WAIT_MS)
|
||||
try {
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve')
|
||||
} catch (error) {
|
||||
// A drain, migration or rehome opened during the wait: Postgres goes back as it was.
|
||||
if (before?.admitMode === 'db') await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
throw error
|
||||
}
|
||||
try {
|
||||
generation = await writeObject(fetchImpl, request, object, accessToken, audit)
|
||||
} catch (error) {
|
||||
// The file was refused outright (412, 4xx): put Postgres back as it was. After a 5xx the
|
||||
// object may be written, so Postgres keeps reserve (sweeps stay off: the safe side).
|
||||
if (error?.notWritten && before?.admitMode === 'db') {
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
}
|
||||
throw error
|
||||
}
|
||||
} else {
|
||||
generation = await writeObject(fetchImpl, request, object, accessToken, audit)
|
||||
}
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_flags_written', cellId: request.cellId, generation }))
|
||||
const deadline = now() + READ_BACK_TIMEOUT_MS
|
||||
let last = runtime.flagsApplied
|
||||
@@ -185,9 +503,22 @@ export async function operateCellFlags(request, dependencies) {
|
||||
// The object is written; a failed poll only delays the read-back until the deadline.
|
||||
continue
|
||||
}
|
||||
if (String(last?.generation) === generation && sameFlags(last.flags, request.object.flags)) {
|
||||
if (String(last?.generation) !== generation) continue
|
||||
// The image predates a switch this object names: it applied the rest and dropped that one.
|
||||
if (Array.isArray(last.ignoredKeys) && last.ignoredKeys.length > 0) {
|
||||
throw new Error(
|
||||
`cell applied generation ${generation} but ignores ${last.ignoredKeys.join(', ')}; ` +
|
||||
'its image predates them, so flip the object back'
|
||||
)
|
||||
}
|
||||
if (sameFlags(last.flags, object.flags)) {
|
||||
const result = { ...plan, written: true, generation, applied: last }
|
||||
log(JSON.stringify({ ...result, event: 'orca_relay_cell_flags_applied_read_back' }))
|
||||
if (admitModeChange === 'db') {
|
||||
await waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log })
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded', cellId: request.cellId, admitMode: 'db' }))
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -205,10 +536,12 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
|
||||
'cell-id',
|
||||
'cell-origin',
|
||||
'project-id',
|
||||
'readiness-local',
|
||||
'ticket-check',
|
||||
'set',
|
||||
'expected-generation',
|
||||
'confirmation'
|
||||
'confirmation',
|
||||
'confirm-reserve',
|
||||
'break-glass',
|
||||
'director-origin'
|
||||
].map((name) => [name, { type: 'string' }])
|
||||
)
|
||||
})
|
||||
|
||||
@@ -1,36 +1,104 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
assertSupportedObject,
|
||||
desiredObject,
|
||||
operateCellFlags,
|
||||
parseCellFlagsRequest,
|
||||
parseFlagChanges,
|
||||
READ_BACK_TIMEOUT_MS,
|
||||
REREGISTER_WAIT_MAX_MS,
|
||||
REREGISTER_WAIT_MIN_MS,
|
||||
reregisterLanes,
|
||||
reregisterWaitMs,
|
||||
WRITE_CONFIRMATION
|
||||
} from './operate-relay-cell-flags.mjs'
|
||||
|
||||
const CELL = 'production-gce-c26'
|
||||
const RESERVE = { 'director-origin': 'https://relay.onorca.dev', 'confirm-reserve': `RESERVE ${CELL}` }
|
||||
const values = (overrides = {}) => ({
|
||||
mode: 'write',
|
||||
'cell-id': CELL,
|
||||
'cell-origin': 'https://c26.relay.onorca.dev',
|
||||
'project-id': 'onorca-cloud',
|
||||
'readiness-local': 'true',
|
||||
'ticket-check': 'off',
|
||||
set: 'readinessLocal=true',
|
||||
'expected-generation': '0',
|
||||
confirmation: `${WRITE_CONFIRMATION} ${CELL}`,
|
||||
...overrides
|
||||
})
|
||||
|
||||
// A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls.
|
||||
function fakeGoogleAndCell({ stored = null, applyAfterPolls = 2, role = 'cell', cellId = CELL } = {}) {
|
||||
const state = { stored, writes: [], polls: 0, applied: { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } } }
|
||||
const SUPPORTED = {
|
||||
readinessLocal: { type: 'boolean' },
|
||||
ticketCheck: { type: 'enum', values: ['off', 'shadow', 'enforce'] },
|
||||
admitMode: { type: 'enum', values: ['db', 'reserve'] },
|
||||
intakePerSec: { type: 'number', min: 0, max: 1_000 },
|
||||
reserveDryRun: { type: 'boolean' },
|
||||
rejectionFence: { type: 'boolean' },
|
||||
readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true },
|
||||
reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true }
|
||||
}
|
||||
|
||||
function fakeGoogleAndCell({
|
||||
stored = null,
|
||||
applyAfterPolls = 2,
|
||||
role = 'cell',
|
||||
cellId = CELL,
|
||||
supportedFlags = SUPPORTED,
|
||||
// Polls that still report admitModeEffective=reserve after a flip back.
|
||||
reregisteringPolls = 0,
|
||||
controls,
|
||||
// Forces admitModeEffective, as a tripped dead-man reports it.
|
||||
effectiveOverride,
|
||||
heartbeatFresh = false,
|
||||
// The director refuses this numbered reserve post (1-based) with this error.
|
||||
refuseReservePost
|
||||
} = {}) {
|
||||
const state = {
|
||||
stored,
|
||||
writes: [],
|
||||
polls: 0,
|
||||
// A cell that has applied whatever object is already there.
|
||||
applied: stored
|
||||
? { generation: Number(stored.generation), flags: stored.object.flags }
|
||||
: { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } },
|
||||
directorAdmitMode: 'db',
|
||||
reservePosts: 0,
|
||||
order: [],
|
||||
reregisteringPolls
|
||||
}
|
||||
const fetchImpl = async (url, init = {}) => {
|
||||
const target = new URL(url)
|
||||
if (target.pathname === '/v1/admin/cell-admit-mode') {
|
||||
const body = JSON.parse(String(init.body))
|
||||
if (body.admitMode === 'reserve') {
|
||||
state.reservePosts += 1
|
||||
if (refuseReservePost?.post === state.reservePosts) {
|
||||
return Response.json({ error: refuseReservePost.error }, { status: 409 })
|
||||
}
|
||||
}
|
||||
if (body.admitMode) {
|
||||
state.directorAdmitMode = body.admitMode
|
||||
state.order.push(`director:${body.admitMode}`)
|
||||
}
|
||||
return Response.json({ v: 1, cellId: body.cellId, admitMode: state.directorAdmitMode, updatedAt: 1, heartbeatFresh })
|
||||
}
|
||||
if (target.pathname === '/v1/admin/runtime-status') {
|
||||
state.polls += 1
|
||||
if (state.stored && state.polls > applyAfterPolls) {
|
||||
state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags }
|
||||
}
|
||||
return Response.json({ v: 1, role, cellId, flagsApplied: state.applied })
|
||||
const reserveApplied = state.applied.flags.admitMode === 'reserve'
|
||||
const effective = reserveApplied || (state.writes.length > 0 && state.reregisteringPolls-- > 0) ? 'reserve' : 'db'
|
||||
return Response.json({
|
||||
v: 1,
|
||||
role,
|
||||
cellId,
|
||||
flagsApplied: state.applied,
|
||||
admitModeEffective: effectiveOverride ?? effective,
|
||||
...(controls === undefined ? {} : { runtime: { controls } }),
|
||||
...(supportedFlags ? { supportedFlags } : {})
|
||||
})
|
||||
}
|
||||
if (target.pathname.startsWith('/upload/')) {
|
||||
const expected = target.searchParams.get('ifGenerationMatch')
|
||||
@@ -41,6 +109,7 @@ function fakeGoogleAndCell({ stored = null, applyAfterPolls = 2, role = 'cell',
|
||||
const generation = String(Number(state.stored?.generation ?? 1000) + 1)
|
||||
state.stored = { generation, object }
|
||||
state.writes.push({ generation, object, metadata })
|
||||
state.order.push(`file:${object.flags.admitMode ?? 'db'}`)
|
||||
return Response.json({ generation })
|
||||
}
|
||||
if (!state.stored) return new Response(null, { status: 404 })
|
||||
@@ -73,7 +142,11 @@ const run = (request, fake, extra = {}) => {
|
||||
test('refuses malformed requests before touching anything', () => {
|
||||
assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ 'ticket-check': 'enforce' })), /ticket-check/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'ticketCheck=always' })), /ticketCheck/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'placerMode=on' })), /unknown switch/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: '' })), /names no switch/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db,admitMode=reserve' })), /twice/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'readTimeoutMarginMs=500' })), /cannot be/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/)
|
||||
assert.throws(
|
||||
@@ -92,7 +165,7 @@ test('a dry run reports the change and writes nothing', async () => {
|
||||
assert.equal(outcome.written, false)
|
||||
assert.equal(fake.state.writes.length, 0)
|
||||
assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
|
||||
assert.deepEqual(lines[0].desired.flags, { readinessLocal: true, ticketCheck: 'off' })
|
||||
assert.deepEqual(lines[0].desired.flags, { readinessLocal: true })
|
||||
})
|
||||
|
||||
test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => {
|
||||
@@ -101,12 +174,12 @@ test('writes with compare-and-swap, records who wrote it, and waits for the cell
|
||||
const outcome = await result
|
||||
assert.equal(outcome.written, true)
|
||||
assert.equal(outcome.generation, '1001')
|
||||
assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true, ticketCheck: 'off' } })
|
||||
assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true } })
|
||||
assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' })
|
||||
assert.deepEqual(fake.state.writes[0].object, {
|
||||
v: 1,
|
||||
cellId: CELL,
|
||||
flags: { readinessLocal: true, ticketCheck: 'off' }
|
||||
flags: { readinessLocal: true }
|
||||
})
|
||||
assert.deepEqual(lines.map((line) => line.event), [
|
||||
'orca_relay_cell_flags_plan',
|
||||
@@ -115,7 +188,7 @@ test('writes with compare-and-swap, records who wrote it, and waits for the cell
|
||||
])
|
||||
// Flip back: the next write must name the generation just written.
|
||||
const back = run(
|
||||
parseCellFlagsRequest(values({ 'readiness-local': 'false', 'expected-generation': '1001' })),
|
||||
parseCellFlagsRequest(values({ set: 'readinessLocal=false', 'expected-generation': '1001' })),
|
||||
fake
|
||||
)
|
||||
assert.equal((await back.result).generation, '1002')
|
||||
@@ -182,3 +255,269 @@ test('fails loudly when the cell does not apply the write within the read-back w
|
||||
await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/)
|
||||
assert.equal(fake.state.writes.length, 1)
|
||||
})
|
||||
|
||||
test('changes only the named switches and keeps every other key the object already has', () => {
|
||||
const current = {
|
||||
v: 1,
|
||||
cellId: CELL,
|
||||
flags: { readinessLocal: true, ticketCheck: 'shadow', readTimeoutMarginMs: 5_000 }
|
||||
}
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ set: 'admitMode=reserve, intakePerSec=40, readTimeoutMarginMs=default', ...RESERVE })
|
||||
)
|
||||
assert.deepEqual(desiredObject(request, current), {
|
||||
v: 1,
|
||||
cellId: CELL,
|
||||
flags: { readinessLocal: true, ticketCheck: 'shadow', admitMode: 'reserve', intakePerSec: 40 }
|
||||
})
|
||||
assert.deepEqual(desiredObject(request, null).flags, { admitMode: 'reserve', intakePerSec: 40 })
|
||||
assert.throws(() => desiredObject(request, { v: 1, cellId: 'production-gce-c27', flags: {} }), /by hand/)
|
||||
assert.deepEqual(parseFlagChanges('rejectionFence=false'), { rejectionFence: false })
|
||||
assert.deepEqual(parseFlagChanges('reregisterInFlight=8'), { reregisterInFlight: 8 })
|
||||
assert.throws(() => parseFlagChanges('reregisterInFlight=32'), /cannot be/)
|
||||
})
|
||||
|
||||
test('a write keeps the switches it did not name, end to end', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
|
||||
})
|
||||
const outcome = await run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result
|
||||
assert.equal(outcome.written, true)
|
||||
assert.deepEqual(fake.state.writes[0].object.flags, { ticketCheck: 'shadow', readinessLocal: true })
|
||||
})
|
||||
|
||||
test('the read-back compares every switch, not only the one this run changed', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
|
||||
})
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
const response = await inner(url, init)
|
||||
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
||||
const body = await response.json()
|
||||
// The cell reports the new generation but still runs the old ticketCheck.
|
||||
if (body.flagsApplied.generation === 8) body.flagsApplied.flags = { readinessLocal: true }
|
||||
return Response.json(body)
|
||||
}
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result,
|
||||
/did not apply generation 8/
|
||||
)
|
||||
})
|
||||
|
||||
test('a switch the cell reports on its own is compared only when the object names it', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
const response = await inner(url, init)
|
||||
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
||||
const body = await response.json()
|
||||
body.flagsApplied.flags = { ...body.flagsApplied.flags, intakePerSec: 50, readTimeoutMarginMs: 10_000 }
|
||||
return Response.json(body)
|
||||
}
|
||||
assert.equal((await run(parseCellFlagsRequest(values()), fake).result).written, true)
|
||||
})
|
||||
|
||||
test('fails the read-back when the cell ignores a switch the object names', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
const response = await inner(url, init)
|
||||
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
||||
const body = await response.json()
|
||||
if (body.flagsApplied.generation === 1001) body.flagsApplied.ignoredKeys = ['admitMode']
|
||||
return Response.json(body)
|
||||
}
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), fake).result,
|
||||
/ignores admitMode/
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses a switch the cell image does not support, before writing anything', async () => {
|
||||
const old = fakeGoogleAndCell({ supportedFlags: null })
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'ticketCheck=enforce' })), old).result,
|
||||
/does not support ticketCheck="enforce"/
|
||||
)
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), old).result,
|
||||
/does not support admitMode/
|
||||
)
|
||||
assert.equal(old.state.writes.length, 0)
|
||||
// An image from before supportedFlags still takes the two switches it always had.
|
||||
assert.equal((await run(parseCellFlagsRequest(values()), old).result).written, true)
|
||||
assert.throws(
|
||||
() => assertSupportedObject({ readTimeoutMarginMs: 1_500.5 }, { supportedFlags: SUPPORTED }),
|
||||
/readTimeoutMarginMs/
|
||||
)
|
||||
assertSupportedObject({ ticketCheck: 'enforce' }, { supportedFlags: SUPPORTED })
|
||||
})
|
||||
|
||||
test('refuses to carry a bad value already in the file', async () => {
|
||||
const bad = fakeGoogleAndCell({
|
||||
supportedFlags: null,
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'enforce' } } }
|
||||
})
|
||||
// The old image voided generation 7 and still runs generation 0.
|
||||
bad.state.applied = { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } }
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), bad).result,
|
||||
/does not support ticketCheck="enforce"/
|
||||
)
|
||||
assert.equal(bad.state.writes.length, 0)
|
||||
})
|
||||
|
||||
const DIRECTOR = { 'director-origin': RESERVE['director-origin'] }
|
||||
|
||||
test('admitMode=reserve needs a typed confirmation, and Postgres records reserve before the file', async () => {
|
||||
assert.throws(
|
||||
() => parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=reserve' })),
|
||||
/--confirm-reserve "RESERVE production-gce-c26"/
|
||||
)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db' })), /--director-origin/)
|
||||
const fake = fakeGoogleAndCell()
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ ...DIRECTOR, set: 'admitMode=reserve', 'confirm-reserve': `RESERVE ${CELL}` })
|
||||
)
|
||||
assert.equal((await run(request, fake).result).written, true)
|
||||
// Recorded, re-checked after every director's cache has expired, then the file.
|
||||
assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'file:reserve'])
|
||||
})
|
||||
|
||||
test('a refused file write after Postgres says reserve puts Postgres back to db', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname.startsWith('/upload/')) return new Response(null, { status: 403 })
|
||||
return await inner(url, init)
|
||||
}
|
||||
const request = parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE }))
|
||||
await assert.rejects(run(request, fake).result, /returned 403/)
|
||||
assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'director:db'])
|
||||
})
|
||||
|
||||
test('repairs an object the cell did not apply when the whole new object is valid', async () => {
|
||||
const voided = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
|
||||
})
|
||||
voided.state.applied = { generation: 3, flags: { readinessLocal: false } }
|
||||
const { result, lines } = run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), voided)
|
||||
assert.equal((await result).written, true)
|
||||
assert.ok(lines.some((line) => line.event === 'orca_relay_cell_flags_repairing_unapplied_object'))
|
||||
})
|
||||
|
||||
test('a flip back records db in Postgres only after the cell has leased every control', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
|
||||
reregisteringPolls: 3
|
||||
})
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
|
||||
assert.equal((await run(request, fake).result).written, true)
|
||||
assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
|
||||
assert.equal(fake.state.reregisteringPolls, -1)
|
||||
})
|
||||
|
||||
test('the flip-back wait scales with the control count and the switch-file pace', () => {
|
||||
assert.equal(reregisterWaitMs({ runtime: { controls: 100 } }), REREGISTER_WAIT_MIN_MS)
|
||||
// 1,500 controls at the default 3 lanes: 1,667 s.
|
||||
assert.equal(reregisterWaitMs({ runtime: { controls: 1_500 } }), 1_666_667)
|
||||
assert.equal(
|
||||
reregisterWaitMs({ runtime: { controls: 1_500 }, flagsApplied: { flags: { reregisterInFlight: 10 } } }),
|
||||
// The cell caps 10 at its US pool of 10 less two.
|
||||
625_000
|
||||
)
|
||||
assert.equal(reregisterWaitMs({ runtime: { controls: 3_000 } }), REREGISTER_WAIT_MAX_MS)
|
||||
})
|
||||
|
||||
test('the wait counts the lanes the cell actually uses', () => {
|
||||
assert.equal(reregisterLanes({ region: 'us-central1' }), 3)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2' }), 5)
|
||||
assert.equal(reregisterLanes({ region: 'us-central1', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 8)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 14)
|
||||
// The pool the cell reports wins over the deployed-size table.
|
||||
assert.equal(reregisterLanes({ region: 'us-central1', databasePoolMax: 22 }), 7)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 12, flagsApplied: { flags: { reregisterInFlight: 16 } } }), 10)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 'x' }), 5)
|
||||
})
|
||||
|
||||
test('a tripped cell takes no write that keeps reserve, only one that also says db', async () => {
|
||||
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
|
||||
const tripped = () => fakeGoogleAndCell({ stored, effectiveOverride: 'db' })
|
||||
for (const set of ['readinessLocal=true', 'reregisterInFlight=4']) {
|
||||
const fake = tripped()
|
||||
const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' }))
|
||||
await assert.rejects(run(request, fake).result, /tripped its dead-man[\s\S]*admitMode=db,reregisterInFlight=N/)
|
||||
assert.deepEqual(fake.state.writes, [])
|
||||
}
|
||||
const fake = tripped()
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ ...DIRECTOR, set: 'admitMode=db,reregisterInFlight=4', 'expected-generation': '7' })
|
||||
)
|
||||
assert.equal((await run(request, fake).result).written, true)
|
||||
assert.deepEqual(fake.state.writes[0].object.flags, { admitMode: 'db', reregisterInFlight: 4 })
|
||||
})
|
||||
|
||||
test('a migration opened during the cache wait puts Postgres back to db', async () => {
|
||||
const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } })
|
||||
const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'admitMode=reserve' }))
|
||||
await assert.rejects(run(request, fake).result, /cell_has_open_migration/)
|
||||
assert.deepEqual(fake.state.order, ['director:reserve', 'director:db'])
|
||||
assert.deepEqual(fake.state.writes, [])
|
||||
})
|
||||
|
||||
test('break glass refuses a cell that answers or that the director still hears', async () => {
|
||||
const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7', 'break-glass': `BREAK GLASS ${CELL}` }
|
||||
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
|
||||
const answering = fakeGoogleAndCell({ stored })
|
||||
await assert.rejects(run(parseCellFlagsRequest(values(glass)), answering).result, /answers runtime-status/)
|
||||
const heard = fakeGoogleAndCell({ stored, heartbeatFresh: true })
|
||||
const inner = heard.fetchImpl
|
||||
heard.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
|
||||
return await inner(url, init)
|
||||
}
|
||||
await assert.rejects(run(parseCellFlagsRequest(values(glass)), heard).result, /still sees heartbeats/)
|
||||
assert.deepEqual(heard.state.writes, [])
|
||||
assert.deepEqual(answering.state.writes, [])
|
||||
})
|
||||
|
||||
test('a flip back that outlasts its wait keeps Postgres on reserve and says how to finish', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
|
||||
reregisteringPolls: Number.POSITIVE_INFINITY,
|
||||
controls: 1_500
|
||||
})
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
|
||||
const { lines, result } = run(request, fake)
|
||||
await assert.rejects(result, /after 1666667 ms[\s\S]*Run again with --set admitMode=db[\s\S]*--set admitMode=db,reregisterInFlight=N/)
|
||||
assert.deepEqual(fake.state.order, ['file:db'])
|
||||
assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667)
|
||||
})
|
||||
|
||||
test('break glass records db for a cell that cannot answer, typed for that cell only', async () => {
|
||||
const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' }
|
||||
assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/)
|
||||
assert.throws(
|
||||
() => parseCellFlagsRequest(values({ ...glass, set: 'admitMode=default,readinessLocal=true', 'break-glass': `BREAK GLASS ${CELL}` })),
|
||||
/on its own/
|
||||
)
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true } } }
|
||||
})
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
|
||||
return await inner(url, init)
|
||||
}
|
||||
const { result, lines } = run(parseCellFlagsRequest(values({ ...glass, 'break-glass': `BREAK GLASS ${CELL}` })), fake)
|
||||
assert.equal((await result).written, true)
|
||||
assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: true })
|
||||
assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
|
||||
assert.equal(lines.at(-1).event, 'orca_relay_cell_admit_mode_break_glass')
|
||||
})
|
||||
|
||||
|
||||
@@ -53,7 +53,9 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
|
||||
try {
|
||||
service = run(gcloudArguments('services', input))
|
||||
} catch (error) {
|
||||
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' }
|
||||
if (error?.code === 'NOT_FOUND') {
|
||||
return { version: input.bootstrap_version, cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const serving = (service.status?.traffic ?? []).filter(
|
||||
@@ -77,12 +79,21 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
|
||||
throw new Error('serving region correction cohort is invalid')
|
||||
}
|
||||
const cohort_percent = cohortSettings[0]?.value ?? '0'
|
||||
// Set by the director deploy (placement) or by hand (shadow cells), never by Terraform: carried
|
||||
// so an apply keeps them, and empty when the serving revision has none.
|
||||
const reserve_placement = servingPlainSetting(revision, 'ORCA_RELAY_RESERVE_PLACEMENT', /^(?:off|dry-run|on)$/)
|
||||
const shadow_seat_feed_cells = servingPlainSetting(
|
||||
revision,
|
||||
'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS',
|
||||
/^(?:all|[A-Za-z0-9_-]+(?:, ?[A-Za-z0-9_-]+)*)$/
|
||||
)
|
||||
const carried = { cohort_percent, reserve_placement, shadow_seat_feed_cells }
|
||||
const references = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter(
|
||||
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
|
||||
)
|
||||
)
|
||||
if (references.length === 0) return { version: input.bootstrap_version, cohort_percent }
|
||||
if (references.length === 0) return { version: input.bootstrap_version, ...carried }
|
||||
const reference = normalizeSecretReference(references[0])
|
||||
if (
|
||||
references.length !== 1 ||
|
||||
@@ -91,7 +102,20 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
|
||||
) {
|
||||
throw new Error('serving regional placement secret reference is invalid')
|
||||
}
|
||||
return { version: reference.version, cohort_percent }
|
||||
return { version: reference.version, ...carried }
|
||||
}
|
||||
|
||||
// One plain value, or none; anything else (a secret, a duplicate, a malformed value) fails the
|
||||
// plan rather than letting an apply drop or reset the setting.
|
||||
function servingPlainSetting(revision, name, valid) {
|
||||
const settings = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter((environment) => environment.name === name)
|
||||
)
|
||||
if (settings.length === 0) return ''
|
||||
if (settings.length > 1 || typeof settings[0].value !== 'string' || !valid.test(settings[0].value)) {
|
||||
throw new Error(`serving ${name} is invalid`)
|
||||
}
|
||||
return settings[0].value
|
||||
}
|
||||
|
||||
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
classifyRelayServiceDescribeFailure,
|
||||
readRelayServingRegionalPlacementVersion
|
||||
@@ -67,7 +69,7 @@ test('reads the exact version from the sole traffic-serving revision', () => {
|
||||
}
|
||||
})
|
||||
|
||||
assert.deepEqual(result, { version: '11', cohort_percent: '0' })
|
||||
assert.deepEqual(result, { version: '11', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
assert.equal(calls[1][3], 'relay-serving')
|
||||
})
|
||||
|
||||
@@ -78,7 +80,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource
|
||||
]) {
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
|
||||
}), { version: '1', cohort_percent: '0' })
|
||||
}), { version: '1', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
}
|
||||
})
|
||||
|
||||
@@ -100,12 +102,12 @@ test('falls back only when the service or setting is absent', () => {
|
||||
notFound.code = 'NOT_FOUND'
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => { throw notFound }
|
||||
}), { version: '7', cohort_percent: '0' })
|
||||
}), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
: { spec: { containers: [{ env: [] }] } }
|
||||
}), { version: '7', cohort_percent: '0' })
|
||||
}), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
})
|
||||
|
||||
test('classifies real absent-service stderr without weakening revision failures', () => {
|
||||
@@ -144,7 +146,7 @@ test('preserves the serving cohort including explicit disable across later Terra
|
||||
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), { version: '11', cohort_percent: value })
|
||||
}), { version: '11', cohort_percent: value, reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
}
|
||||
})
|
||||
|
||||
@@ -164,3 +166,44 @@ test('fails closed on malformed, secret-backed or duplicate cohorts rather than
|
||||
}), /cohort is invalid/)
|
||||
}
|
||||
})
|
||||
|
||||
test('carries the deploy-owned placement and shadow cells, so an apply keeps them', () => {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push(
|
||||
{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' },
|
||||
{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'production-gce-c1,production-gce-c2' }
|
||||
)
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), {
|
||||
version: '11',
|
||||
cohort_percent: '0',
|
||||
reserve_placement: 'on',
|
||||
shadow_seat_feed_cells: 'production-gce-c1,production-gce-c2'
|
||||
})
|
||||
})
|
||||
|
||||
test('fails closed on a malformed, secret-backed or duplicate placement or shadow-cell setting', () => {
|
||||
const cases = [
|
||||
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'yes' }],
|
||||
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' }, { name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'off' }],
|
||||
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', valueFrom: { secretKeyRef: { name: 'x', key: '1' } } }],
|
||||
[{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'c1;c2' }],
|
||||
[{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: '' }]
|
||||
]
|
||||
for (const settings of cases) {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push(...settings)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), /is invalid/)
|
||||
}
|
||||
})
|
||||
|
||||
test('Terraform sets the carried settings only when the serving revision has them', () => {
|
||||
const terraform = readFileSync(fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), 'utf8')
|
||||
const block = /dynamic "env" \{([\s\S]*?)\n {6}\}/.exec(terraform)?.[1] ?? ''
|
||||
assert.match(block, /ORCA_RELAY_RESERVE_PLACEMENT *= data\.external\.relay_serving_regional_placement_version\.result\.reserve_placement/)
|
||||
assert.match(block, /ORCA_RELAY_SHADOW_SEAT_FEED_CELLS *= data\.external\.relay_serving_regional_placement_version\.result\.shadow_seat_feed_cells/)
|
||||
assert.match(block, /if value != ""/)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { parseArguments as parseBlueGreen } from './deploy-relay-blue-green.mjs'
|
||||
import { parseArguments as parseGceCandidate } from './deploy-relay-gce-candidate.mjs'
|
||||
import { RELAY_WORKFLOW_DIRECTORY, readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
// Every workflow's real argv, literals as written: a flag combination the parser refuses
|
||||
// (staging's lone --admin-audience once was one) fails here instead of on a release.
|
||||
|
||||
const PROJECT = 'orca-test-project'
|
||||
const DIGEST = `sha256:${'a'.repeat(64)}`
|
||||
|
||||
// Shell-expanded values only; anything a workflow writes literally is parsed as written.
|
||||
const SAMPLES = {
|
||||
project: PROJECT,
|
||||
image: `relay@${DIGEST}`,
|
||||
'predecessor-image-digest': DIGEST,
|
||||
'director-cells-json': JSON.stringify([
|
||||
{ id: 'cell-a', url: 'https://cell-a.example.test', region: 'us-central1', capacityRequests: 10, initiallyEnabled: true }
|
||||
]),
|
||||
'regional-placement-secret-version': '3',
|
||||
'min-instances': '1',
|
||||
'max-instances': '5',
|
||||
'prune-revisions': 'false',
|
||||
'bootstrap-runtime-identity': 'false',
|
||||
'expected-rehome-generation': '4',
|
||||
'region-correction-cohort-percent': '10',
|
||||
'rehome-audience': 'https://relay.example.test/v1/admin/host-drain',
|
||||
'director-origin': 'https://relay.example.test',
|
||||
'admin-audience': 'https://relay.example.test/v1/admin/drain',
|
||||
'reserve-placement': 'preserve',
|
||||
'shadow-seat-feed-cells': 'preserve',
|
||||
mode: 'audit'
|
||||
}
|
||||
|
||||
function sample(flag) {
|
||||
if (flag in SAMPLES) return SAMPLES[flag]
|
||||
if (flag.endsWith('service-account')) return `${flag.slice(0, 20).replace(/-$/, '')}@${PROJECT}.iam.gserviceaccount.com`
|
||||
return `sample-${flag}`
|
||||
}
|
||||
|
||||
function invocations(source, script) {
|
||||
const lines = source.split('\n')
|
||||
const found = []
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
if (!lines[index].includes(`dev/scripts/${script}`)) continue
|
||||
const argv = []
|
||||
let line = lines[index]
|
||||
while (line.trimEnd().endsWith('\\')) {
|
||||
line = lines[++index]
|
||||
const match = /^\s*(--[a-z0-9-]+)\s+(.+?)\s*\\?\s*$/.exec(line)
|
||||
if (!match) throw new Error(`unparsed ${script} argument line: ${line.trim()}`)
|
||||
const [, flag, raw] = match
|
||||
// A trailing `)"` closes the command substitution the invocation sits in.
|
||||
const value = raw.replace(/\)"$/, '').replace(/^"(.*)"$/, '$1')
|
||||
argv.push(flag, value.includes('$') ? sample(flag.slice(2)) : value)
|
||||
}
|
||||
found.push(argv)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
const SCRIPTS = [
|
||||
['deploy-relay-blue-green.mjs', parseBlueGreen],
|
||||
['deploy-relay-gce-candidate.mjs', parseGceCandidate]
|
||||
]
|
||||
|
||||
for (const [script, parse] of SCRIPTS) {
|
||||
test(`every workflow's ${script} argv parses`, () => {
|
||||
let count = 0
|
||||
for (const file of readdirSync(RELAY_WORKFLOW_DIRECTORY).filter((name) => name.endsWith('.yml'))) {
|
||||
for (const argv of invocations(readFileSync(new URL(file, RELAY_WORKFLOW_DIRECTORY), 'utf8'), script)) {
|
||||
count++
|
||||
assert.doesNotThrow(() => parse(argv), `${file}: ${argv.join(' ')}`)
|
||||
}
|
||||
}
|
||||
assert.ok(count > 0, `no workflow runs ${script}`)
|
||||
})
|
||||
}
|
||||
|
||||
test('the production and staging director workflows are among the parsed', () => {
|
||||
for (const name of ['deploy-relay-production-director.yml', 'deploy-relay-staging.yml']) {
|
||||
assert.equal(invocations(readRelayWorkflow(name), 'deploy-relay-blue-green.mjs').length, 1, name)
|
||||
}
|
||||
})
|
||||
@@ -211,6 +211,8 @@ export function directorDeployInputs({ imageDigest, predecessorDigest, rehomeGen
|
||||
'image-digest': imageDigest,
|
||||
'regional-placement-mode': 'preserve',
|
||||
'region-correction-cohort-percent': 'preserve',
|
||||
'reserve-placement': 'preserve',
|
||||
'shadow-seat-feed-cells': 'preserve',
|
||||
'prune-incompatible-revisions': 'false',
|
||||
'expected-rehome-generation': String(rehomeGeneration),
|
||||
'bootstrap-runtime-identity': 'false',
|
||||
|
||||
@@ -174,6 +174,22 @@ resource "google_cloud_run_v2_service" "relay" {
|
||||
value = data.external.relay_serving_regional_placement_version.result.cohort_percent
|
||||
}
|
||||
|
||||
# The director deploy owns these (shadow cells are set by hand): an apply keeps whatever the
|
||||
# serving revision has, and adds nothing when it has none.
|
||||
dynamic "env" {
|
||||
for_each = {
|
||||
for name, value in {
|
||||
ORCA_RELAY_RESERVE_PLACEMENT = data.external.relay_serving_regional_placement_version.result.reserve_placement
|
||||
ORCA_RELAY_SHADOW_SEAT_FEED_CELLS = data.external.relay_serving_regional_placement_version.result.shadow_seat_feed_cells
|
||||
} : name => value if value != ""
|
||||
}
|
||||
|
||||
content {
|
||||
name = env.key
|
||||
value = env.value
|
||||
}
|
||||
}
|
||||
|
||||
ports {
|
||||
container_port = 8080
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user