diff --git a/.github/workflows/cloud-deploy-relay-production-director.yml b/.github/workflows/cloud-deploy-relay-production-director.yml index 07e1e74ec3c..6094f25ccaf 100644 --- a/.github/workflows/cloud-deploy-relay-production-director.yml +++ b/.github/workflows/cloud-deploy-relay-production-director.yml @@ -18,6 +18,17 @@ on: required: true default: preserve type: string + reserve-placement: + description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve + required: true + default: preserve + type: choice + options: [preserve, 'off', dry-run, 'on'] + shadow-seat-feed-cells: + description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused + required: true + default: preserve + type: string prune-incompatible-revisions: description: Retain only the newly verified serving and rollback revisions required: true @@ -68,6 +79,8 @@ jobs: IMAGE_DIGEST: ${{ inputs.image-digest }} REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }} REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }} + RESERVE_PLACEMENT: ${{ inputs.reserve-placement }} + SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }} PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }} # Floor the served revision must keep, matching relay_min_instances in # environments/production.tfvars. This gate only fails a bad deploy; Terraform @@ -228,7 +241,9 @@ jobs: --prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \ --release-id "${RELEASE_ID}" \ --regional-placement-secret-version "${target_version}" \ - --region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" + --region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" \ + --reserve-placement "${RESERVE_PLACEMENT}" \ + --shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}" echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/cloud-deploy-relay-staging.yml b/.github/workflows/cloud-deploy-relay-staging.yml index 7ddfb9d7c73..bd82fbfb851 100644 --- a/.github/workflows/cloud-deploy-relay-staging.yml +++ b/.github/workflows/cloud-deploy-relay-staging.yml @@ -7,6 +7,17 @@ on: description: Exact checked-in production Relay sha256 digest to deploy required: true type: string + reserve-placement: + description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve + required: true + default: preserve + type: choice + options: [preserve, 'off', dry-run, 'on'] + shadow-seat-feed-cells: + description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused + required: true + default: preserve + type: string permissions: contents: read @@ -87,6 +98,11 @@ jobs: test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}" - name: Deploy director blue/green + env: + # An image change reads every cell's admit mode before the traffic move. + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + RESERVE_PLACEMENT: ${{ inputs.reserve-placement }} + SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }} run: | regional_version="$(gcloud secrets versions describe latest \ --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \ @@ -104,6 +120,9 @@ jobs: --asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \ --regional-placement-secret-version "${regional_version}" \ --min-instances 0 \ + --admin-audience https://relay-staging.onorca.dev/v1/admin/drain \ + --reserve-placement "${RESERVE_PLACEMENT}" \ + --shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" \ --release-id "${RELEASE_ID}" - name: Smoke director health diff --git a/.github/workflows/cloud-operate-relay-production-cell-flags.yml b/.github/workflows/cloud-operate-relay-production-cell-flags.yml index 74576e0f6c7..a38b65978dc 100644 --- a/.github/workflows/cloud-operate-relay-production-cell-flags.yml +++ b/.github/workflows/cloud-operate-relay-production-cell-flags.yml @@ -15,18 +15,26 @@ on: description: Exact cell id, e.g. production-gce-c26 required: true type: string - readiness-local: - description: /ready answers from the cell alone (local readiness) + break-glass: + description: >- + Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as + BREAK GLASS ; records db without the cell's read-back + required: false + default: '' + type: string + confirm-reserve: + description: Required for admitMode=reserve, typed as RESERVE + required: false + default: '' + type: string + set: + description: >- + Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a + key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck, + admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs, + reregisterInFlight required: true - default: 'false' - type: choice - options: ['false', 'true'] - ticket-check: - description: Shadow-check the director's assignment lease on host hellos - required: true - default: 'off' - type: choice - options: ['off', 'shadow'] + type: string expected-generation: description: Exact current object generation from a dry run, or 0 if none exists required: true @@ -55,15 +63,17 @@ jobs: operate: if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} runs-on: ubuntu-latest - timeout-minutes: 10 + # A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls. + timeout-minutes: 60 environment: production env: MODE: ${{ inputs.mode }} CELL_ID: ${{ inputs.cell-id }} - READINESS_LOCAL: ${{ inputs.readiness-local }} - TICKET_CHECK: ${{ inputs.ticket-check }} + FLAG_CHANGES: ${{ inputs.set }} EXPECTED_GENERATION: ${{ inputs.expected-generation }} CONFIRMATION: ${{ inputs.confirmation }} + CONFIRM_RESERVE: ${{ inputs.confirm-reserve }} + BREAK_GLASS: ${{ inputs.break-glass }} GCP_PROJECT_ID: onorca-cloud steps: - uses: actions/checkout@v4 @@ -95,7 +105,9 @@ jobs: --mode "${MODE}" --cell-id "${CELL_ID}" \ --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ --project-id "${GCP_PROJECT_ID}" \ - --readiness-local "${READINESS_LOCAL}" --ticket-check "${TICKET_CHECK}" \ + --set "${FLAG_CHANGES}" \ + --director-origin "https://relay.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \ + --break-glass "${BREAK_GLASS}" \ --expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \ | tee "${RUNNER_TEMP}/relay-cell-flags.jsonl" diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs index ddfe2bce8fc..72bb6355986 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs @@ -11,6 +11,11 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET = export const DIRECTOR_REGIONAL_PLACEMENT_ENV = 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT' +// Step 5 placement on the directors; only this deploy changes it, so the reserve guard sees it. +export const DIRECTOR_RESERVE_PLACEMENT_ENV = 'ORCA_RELAY_RESERVE_PLACEMENT' +const RESERVE_PLACEMENT_MODES = ['off', 'dry-run', 'on'] +// The cells whose feeds a director polls; a reserve cell outside it has no placing director. +export const DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV = 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS' export const DIRECTOR_REHOME_IDENTITY_ENV = 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT' export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE' @@ -250,6 +255,12 @@ export function directorDeploymentEnvironment(config) { config['region-correction-cohort-percent'] !== 'preserve') { environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent']) } + if (config['reserve-placement'] !== undefined && config['reserve-placement'] !== 'preserve') { + environment[DIRECTOR_RESERVE_PLACEMENT_ENV] = config['reserve-placement'] + } + if (config['shadow-seat-feed-cells'] !== undefined && config['shadow-seat-feed-cells'] !== 'preserve') { + environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] = shadowSeatFeedCellsSetting(config['shadow-seat-feed-cells']) + } const serviceAccount = projectServiceAccount(config, 'capacity-service-account') const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account') const rehomeDirectorServiceAccount = projectServiceAccount( @@ -271,6 +282,16 @@ export function directorDeploymentEnvironment(config) { return environment } +// `all` or a comma list of cell ids, as the director's config reads it. +function shadowSeatFeedCellsSetting(value) { + if (value === 'all') return value + const cells = value.split(',').map((cell) => cell.trim()) + if (cells.length > 100 || cells.some((cell) => !/^[a-z][a-z0-9-]{0,39}$/.test(cell))) { + throw new Error('--shadow-seat-feed-cells must be preserve, all, or a comma list of cell ids') + } + return cells.join(',') +} + export function environmentUpdateValue(environment) { const entries = Object.entries(environment) if (entries.every(([key, value]) => !key.includes(',') && !value.includes(','))) { @@ -315,11 +336,19 @@ export function parseArguments(argv) { values['rehome-audience'] !== undefined || values['expected-rehome-generation'] !== undefined || values['rehome-control-origin'] !== undefined || - values['region-correction-cohort-percent'] !== undefined + values['region-correction-cohort-percent'] !== undefined || + values['reserve-placement'] !== undefined || + values['shadow-seat-feed-cells'] !== undefined ) { throw new Error('director configuration arguments require --role director') } } + if ( + values['reserve-placement'] !== undefined && + !['preserve', ...RESERVE_PLACEMENT_MODES].includes(values['reserve-placement']) + ) { + throw new Error('--reserve-placement must be preserve, off, dry-run or on') + } if ( values.role === 'director' && values['capacity-cell-id'] !== undefined && @@ -378,12 +407,14 @@ export function parseArguments(argv) { throw new Error('--rehome-audience must be an exact host-drain HTTPS URL') } } - const controlArguments = [ - 'expected-rehome-generation', - 'rehome-control-origin', - 'admin-audience' - ].map((key) => values[key] !== undefined) - if (controlArguments.some(Boolean) && !controlArguments.every(Boolean)) { + // --admin-audience alone serves the reserve guard's reads; durable verification needs all three. + const controlArguments = ['expected-rehome-generation', 'rehome-control-origin'].map( + (key) => values[key] !== undefined + ) + if ( + controlArguments.some(Boolean) && + (!controlArguments.every(Boolean) || values['admin-audience'] === undefined) + ) { throw new Error('durable rehome verification arguments must be configured together') } if ( @@ -429,9 +460,35 @@ export function suppliedAdminIdentityToken(environment = process.env) { return token } +// The unverified `email` claim, only to name the caller in an error; the director verifies it. +export function adminTokenEmail(token) { + try { + const payload = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString('utf8')) + return typeof payload?.email === 'string' ? payload.email : undefined + } catch { + return undefined + } +} + +// cell-admit-mode answers only the revision's deploy identity, so a drifted one (a revision that +// trusts another account than the workflow's) fails here by name instead of as a bare 401. +export function assertDirectorTrustsAdminCaller(token, environment, label) { + const trusted = environment.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT + const caller = adminTokenEmail(token) + if (trusted === undefined || caller === undefined || caller === trusted) return + throw new Error( + `the ${label} director trusts ${trusted} for deploy admin reads, but this deploy authenticates as ${caller}; ` + + 'run as that identity or correct the revision\'s ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT' + ) +} + function adminIdentityToken(config) { + const supplied = suppliedAdminIdentityToken() + if (supplied === null && !config['admin-audience']) { + throw new Error('admin reads need --admin-audience or a supplied ORCA_RELAY_ADMIN_ID_TOKEN') + } return ( - suppliedAdminIdentityToken() ?? + supplied ?? commandText(['auth', 'print-identity-token', `--audiences=${config['admin-audience']}`], { sensitive: true }) @@ -721,6 +778,108 @@ export async function assertRegionalRehomeDisabled( return result.control } +// `undefined` when the director predates the reserve census (its router 404s the route). +export async function readReserveCells(config, origin, cellIds, fetchImpl = fetch) { + const reserve = [] + for (const cellId of cellIds) { + const response = await fetchImpl(`${origin}/v1/admin/cell-admit-mode`, { + method: 'POST', + headers: { + authorization: `Bearer ${adminIdentityToken(config)}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1, cellId }), + signal: AbortSignal.timeout(30_000) + }) + const result = await response.json().catch(() => null) + if (response.status === 404 && result === null) return undefined + if (!response.ok || result?.v !== 1 || !['db', 'reserve'].includes(result.admitMode)) { + throw new Error(`cell admit mode read failed for ${cellId}: ${result?.error ?? response.status}`) + } + if (result.admitMode === 'reserve') reserve.push(cellId) + } + return reserve +} + +// A director without the census places hosts on reserve cells from stale rows, so traffic may +// move between revisions that differ in it only while every cell is db. +export function assertReserveCompatibleTrafficMove(servingReserve, candidateReserve) { + if ((servingReserve === undefined) === (candidateReserve === undefined)) return + const reserve = servingReserve ?? candidateReserve + if (reserve.length > 0) { + throw new Error( + `director revisions differ in reserve-mode support while cells are reserve (${reserve.join(', ')}); flip them to db first` + ) + } +} + +// With placement off no director books a reserve cell, so each one's dead-man trips (60-120 s) +// and every host re-registers at once: the cells go to db first, on purpose. +export function assertReservePlacementKept(servingPlacement, candidatePlacement, reserve) { + if (servingPlacement !== 'on' || candidatePlacement === 'on') return + if (reserve !== undefined && reserve.length > 0) { + throw new Error( + `reserve placement would turn ${candidatePlacement} while cells are reserve (${reserve.join(', ')}); flip them to db first` + ) + } +} + +// Placement as the director built it (runtime-status), not as its env asks; an image from before +// the field reports nothing, and only its env can say. +export async function readDirectorPlacement(config, origin, fetchImpl = fetch) { + const response = await fetchImpl(`${origin}/v1/admin/runtime-status`, { + method: 'POST', + headers: { + authorization: `Bearer ${adminIdentityToken(config)}`, + 'content-type': 'application/json' + }, + body: JSON.stringify({ v: 1 }), + signal: AbortSignal.timeout(30_000) + }) + const result = await response.json().catch(() => null) + if (!response.ok || result?.v !== 1 || result.role !== 'director') { + throw new Error(`director runtime status failed at ${origin}: ${result?.error ?? response.status}`) + } + if (result.reservePlacement === undefined) return 'unreported' + if (!RESERVE_PLACEMENT_MODES.includes(result.reservePlacement)) { + throw new Error(`director at ${origin} reports reserve placement ${result.reservePlacement}`) + } + return result.reservePlacement +} + +const shadowCellSet = (value) => + (value ?? '').trim() === 'all' + ? 'all' + : new Set( + (value ?? '') + .split(',') + .map((cell) => cell.trim()) + .filter(Boolean) + ) + +// A reserve cell the directors stop polling gets no booking and no reserver poll: its dead-man +// trips. Shrinking the shadow-cell list past one is refused until that cell is db. +export function assertShadowCellsKeepReserveCells(servingCells, candidateCells, reserve) { + if (reserve === undefined || reserve.length === 0) return + const before = shadowCellSet(servingCells) + const after = shadowCellSet(candidateCells) + if (after === 'all') return + const dropped = reserve.filter((cell) => (before === 'all' || before.has(cell)) && !after.has(cell)) + if (dropped.length > 0) { + throw new Error( + `the shadow seat-feed cells would drop reserve cells (${dropped.join(', ')}); flip them to db first` + ) + } +} + +function directorCellIds(...environments) { + const ids = new Set() + for (const environment of environments) { + for (const cell of JSON.parse(environment.ORCA_RELAY_CELLS_JSON ?? '[]')) ids.add(cell.id) + } + return [...ids] +} + function assertDirectorRevisionIdentity(revision, config) { const expectedRuntimeServiceAccount = config['runtime-service-account'] if ( @@ -741,6 +900,9 @@ export async function deployDirector(config, tag, overrides = {}) { updateTraffic, waitForHealth, assertRegionalRehomeDisabled, + readReserveCells, + readDirectorPlacement, + adminIdentityToken, ...overrides } // Why: gcloud does not carry minScale onto a new revision, and the candidate below takes @@ -801,6 +963,12 @@ export async function deployDirector(config, tag, overrides = {}) { deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent( currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0' ) + if (currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] !== undefined) { + deploymentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] ??= currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] + } + if (currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] !== undefined) { + deploymentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] ??= currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] + } if (config['region-correction-cohort-percent'] !== undefined && config['region-correction-cohort-percent'] !== 'preserve' && config['expected-rehome-generation'] === undefined) { @@ -911,6 +1079,46 @@ export async function deployDirector(config, tag, overrides = {}) { } await operations.waitForHealth(candidate.origin, requiredCapacityProtocol) await verifyRehomeDisabled(candidate.origin) + // The same image and the same placement and shadow-cell settings support the same things; + // anything else needs every cell's admit mode (and the admin token that reads it). + const sameImage = + servingRevision.spec?.containers?.[0]?.image?.split('@').at(-1) === config.image?.split('@').at(-1) + const sameSetting = (name) => (currentEnvironment[name] ?? '') === (environment[name] ?? '') + if (sameImage && sameSetting(DIRECTOR_RESERVE_PLACEMENT_ENV) && sameSetting(DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV)) { + console.warn(JSON.stringify({ event: 'director_reserve_support_checked', skipped: 'same image and settings' })) + } else { + const servingOrigin = initialService.status?.url + if (!servingOrigin) throw new Error('relay service reports no URL for the serving revision') + const token = operations.adminIdentityToken(config) + assertDirectorTrustsAdminCaller(token, currentEnvironment, 'serving') + assertDirectorTrustsAdminCaller(token, environment, 'candidate') + const cellIds = directorCellIds(currentEnvironment, environment) + const servingReserve = await operations.readReserveCells(config, servingOrigin, cellIds) + const candidateReserve = await operations.readReserveCells(config, candidate.origin, cellIds) + const reserve = candidateReserve ?? servingReserve + // Placement as each director built it, which an env string alone does not say. + const placementAt = async (origin, env) => { + const placement = await operations.readDirectorPlacement(config, origin) + return placement === 'unreported' ? (env[DIRECTOR_RESERVE_PLACEMENT_ENV] ?? 'off') : placement + } + const servingPlacement = await placementAt(servingOrigin, currentEnvironment) + const candidatePlacement = await placementAt(candidate.origin, environment) + assertReserveCompatibleTrafficMove(servingReserve, candidateReserve) + assertReservePlacementKept(servingPlacement, candidatePlacement, reserve) + assertShadowCellsKeepReserveCells( + currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], + environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], + reserve + ) + console.warn( + JSON.stringify({ + event: 'director_reserve_support_checked', + reservePlacement: { serving: servingPlacement, candidate: candidatePlacement }, + serving: servingReserve === undefined ? 'unsupported' : servingReserve, + candidate: candidateReserve === undefined ? 'unsupported' : candidateReserve + }) + ) + } operations.updateTraffic(config, [`--to-tags=${tag}=100`]) promoted = true removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG])) diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs index a68ce50e912..782ace852c7 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs @@ -13,13 +13,19 @@ import { DIRECTOR_REHOME_AUDIENCE_ENV, DIRECTOR_REHOME_IDENTITY_ENV, assertRegionalRehomeDisabled, + assertReserveCompatibleTrafficMove, + assertShadowCellsKeepReserveCells, deployDirector, + DIRECTOR_RESERVE_PLACEMENT_ENV, directorDeploymentEnvironment, + DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV, directorCellSetAddition, directorStartupProbeArguments, directorTopologyChange, environmentUpdateValue, parseArguments, + readDirectorPlacement, + readReserveCells, revisionEnvironment, revisionSecretEnvironment, suppliedAdminIdentityToken, @@ -418,6 +424,9 @@ test('requires exactly one active revision and reads queried tag metadata', () = assert.deepEqual(trafficTags(service), ['candidate-123']) }) +const adminToken = (email) => + `e30.${Buffer.from(JSON.stringify(email ? { email } : {})).toString('base64url')}.sig` + function directorHarness({ deployFailure, deleteFailure, @@ -428,7 +437,11 @@ function directorHarness({ // Reproduces gcloud dropping minScale from a newly created revision. dropRequestedMinimum = false, servingServiceAccount, - servingImageDigest = `sha256:${'f'.repeat(64)}` + servingImageDigest = `sha256:${'f'.repeat(64)}`, + reserveCells = {}, + servingEnvironment = {}, + placements = {}, + callerEmail } = {}) { const state = { activeRevision: 'relay-00001-old', @@ -439,7 +452,8 @@ function directorHarness({ 'relay-00001-old', { env: { - ORCA_RELAY_ROLE: 'director' + ORCA_RELAY_ROLE: 'director', + ...servingEnvironment }, secrets: { [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { @@ -462,6 +476,7 @@ function directorHarness({ const operations = { describeService: () => ({ status: { + url: 'https://relay-hash-uc.a.run.app', traffic: [ { percent: 100, revisionName: state.activeRevision }, ...[...state.tags].map(([tag, revisionName]) => ({ @@ -501,7 +516,8 @@ function directorHarness({ deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => { const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new` state.revisions.set(revision, { - env: { ORCA_RELAY_ROLE: 'director', ...env }, + // gcloud --update-env-vars keeps every variable the deploy does not name. + env: { ORCA_RELAY_ROLE: 'director', ...servingEnvironment, ...env }, secrets: { [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, @@ -544,7 +560,10 @@ function directorHarness({ }, waitForHealth: async (_origin, connectionCapacityProtocol) => { healthProtocols.push(connectionCapacityProtocol) - } + }, + readReserveCells: async (_config, origin) => reserveCells[origin], + readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported', + adminIdentityToken: () => adminToken(callerEmail) } return { state, removed, healthProtocols, operations } } @@ -854,3 +873,206 @@ test('sets a reviewed cohort only behind repeated disabled-control verification' assert.ok(verified >= 2) assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1') }) + +test('director deploy refuses a traffic move between reserve-support revisions while a cell is reserve', async () => { + const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' + const oldServing = directorHarness({ + reserveCells: { [candidateOrigin]: ['production-gce-c3'] } + }) + await assert.rejects( + deployDirector({}, 'candidate-new', oldServing.operations), + /differ in reserve-mode support while cells are reserve \(production-gce-c3\)/ + ) + assert.equal(oldServing.state.activeRevision, 'relay-00001-old') + const rollbackToOld = directorHarness({ + reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'] } + }) + await assert.rejects(deployDirector({}, 'candidate-new', rollbackToOld.operations), /flip them to db first/) + assert.equal(rollbackToOld.state.activeRevision, 'relay-00001-old') + const allDb = directorHarness({ reserveCells: { [candidateOrigin]: [] } }) + await deployDirector({}, 'candidate-new', allDb.operations) + assert.equal(allDb.state.activeRevision, 'relay-00003-new') +}) + +test('two revisions that both read the census may move traffic with reserve cells', () => { + assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(['c3'], ['c3'])) + assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(undefined, undefined)) + assert.throws(() => assertReserveCompatibleTrafficMove(undefined, ['c3']), /flip them to db first/) +}) + +test('reads a census-less director as unsupported and fails closed on any other refusal', async () => { + const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN + process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' + const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' } + try { + assert.equal( + await readReserveCells(config, 'https://old.example.test', ['c1'], async () => + new Response('404 Not Found', { status: 404 }) + ), + undefined + ) + const modes = { c1: 'db', c2: 'reserve' } + assert.deepEqual( + await readReserveCells(config, 'https://new.example.test', ['c1', 'c2'], async (url, init) => { + assert.equal(url, 'https://new.example.test/v1/admin/cell-admit-mode') + const { cellId } = JSON.parse(init.body) + return new Response(JSON.stringify({ v: 1, cellId, admitMode: modes[cellId], updatedAt: 1 })) + }), + ['c2'] + ) + await assert.rejects( + readReserveCells(config, 'https://new.example.test', ['c1'], async () => + new Response(JSON.stringify({ error: 'invalid_token' }), { status: 401 }) + ), + /cell admit mode read failed for c1: invalid_token/ + ) + } finally { + if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN + else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment + } +}) + +test('a same-image deploy reads no admit modes, and keeps the serving reserve placement', async () => { + const servingImageDigest = `sha256:${'f'.repeat(64)}` + const harness = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' } }) + let reads = 0 + harness.operations.readReserveCells = async () => { + reads += 1 + return [] + } + await deployDirector({ image: `relay@${servingImageDigest}` }, 'candidate-new', harness.operations) + assert.equal(reads, 0) + assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'on') +}) + +test('turning reserve placement off is refused while any cell is reserve, even on the same image', async () => { + const servingImageDigest = `sha256:${'f'.repeat(64)}` + const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' + const reserved = directorHarness({ + servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, + reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] }, + placements: { 'https://relay-hash-uc.a.run.app': 'on', [candidateOrigin]: 'off' } + }) + await assert.rejects( + deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', reserved.operations), + /reserve placement would turn off while cells are reserve \(production-gce-c3\); flip them to db first/ + ) + assert.equal(reserved.state.activeRevision, 'relay-00001-old') + const allDb = directorHarness({ + servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, + reserveCells: { 'https://relay-hash-uc.a.run.app': [], [candidateOrigin]: [] } + }) + await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', allDb.operations) + assert.equal(allDb.state.revisions.get(allDb.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off') +}) + +test('--reserve-placement takes only a known mode, and only for a director', () => { + const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'a'.repeat(64)}`, '--release-id', '1'] + assert.equal(parseArguments([...base, '--role', 'director', '--reserve-placement', 'on'])['reserve-placement'], 'on') + assert.throws(() => parseArguments([...base, '--role', 'director', '--reserve-placement', 'yes']), /must be preserve, off, dry-run or on/) + assert.throws( + () => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--reserve-placement', 'on']), + /require --role director/ + ) + assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'preserve' })[DIRECTOR_RESERVE_PLACEMENT_ENV], undefined) + assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'dry-run' })[DIRECTOR_RESERVE_PLACEMENT_ENV], 'dry-run') +}) + +test('the guard reads placement as built, so a director that never placed does not block turning it off', async () => { + const servingImageDigest = `sha256:${'f'.repeat(64)}` + const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' + // Configured on, but the serving director reports off: it was never feeding the dead-man. + const harness = directorHarness({ + servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, + reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] }, + placements: { 'https://relay-hash-uc.a.run.app': 'off', [candidateOrigin]: 'off' } + }) + await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations) + assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off') +}) + +test('a shadow-cell change on the same image is checked, and a shrink past a reserve cell is refused', () => { + assert.throws( + () => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', ['production-gce-c3']), + /would drop reserve cells \(production-gce-c3\); flip them to db first/ + ) + assert.throws(() => assertShadowCellsKeepReserveCells('all', 'production-gce-c1', ['production-gce-c3']), /production-gce-c3/) + assert.throws(() => assertShadowCellsKeepReserveCells('all', undefined, ['production-gce-c3']), /production-gce-c3/) + assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'all', ['production-gce-c3'])) + assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', [])) + // A reserve cell the serving list never polled is not being dropped by this deploy. + assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'production-gce-c1', ['production-gce-c3'])) +}) + +test('reads a director placement from runtime-status, and reports an image without the field', async () => { + const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN + process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' + const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' } + try { + const answer = (body) => async () => new Response(JSON.stringify({ v: 1, role: 'director', ...body })) + assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'on' })), 'on') + assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({})), 'unreported') + await assert.rejects(readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'maybe' })), /reports reserve placement maybe/) + await assert.rejects( + readDirectorPlacement(config, 'https://d.example.test', async () => new Response('{}', { status: 401 })), + /runtime status failed/ + ) + } finally { + if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN + else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment + } +}) + +// Staging's director trusted gha-deploy while the workflow authenticated as gha-relay: a bare 401. +test('the guard names the deploy identity a drifted director trusts instead of reading into a 401', async () => { + const servingImageDigest = `sha256:${'f'.repeat(64)}` + const harness = directorHarness({ + servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-deploy@p.iam.gserviceaccount.com' }, + callerEmail: 'gha-relay@p.iam.gserviceaccount.com' + }) + await assert.rejects( + deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations), + /serving director trusts gha-deploy@p\.iam\.gserviceaccount\.com for deploy admin reads, but this deploy authenticates as gha-relay@p\.iam\.gserviceaccount\.com/ + ) + const trusted = directorHarness({ + servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-relay@p.iam.gserviceaccount.com' }, + callerEmail: 'gha-relay@p.iam.gserviceaccount.com' + }) + await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', trusted.operations) +}) + +test('a director deploy takes --admin-audience alone for the reserve guard', () => { + const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'e'.repeat(64)}`, '--role', 'director', '--release-id', 'x'] + const audience = ['--admin-audience', 'https://relay.example.test'] + assert.equal(parseArguments([...base, ...audience])['admin-audience'], 'https://relay.example.test') + assert.throws( + () => parseArguments([...base, '--expected-rehome-generation', '1', '--rehome-control-origin', 'https://relay.example.test']), + /durable rehome verification arguments must be configured together/ + ) + assert.throws( + () => parseArguments([...base, ...audience, '--expected-rehome-generation', '1']), + /durable rehome verification arguments must be configured together/ + ) +}) + +test('--shadow-seat-feed-cells sets the director setting, and preserve keeps the serving one', async () => { + assert.equal( + directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'c1, c2' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], + 'c1,c2' + ) + assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'all' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'all') + assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'preserve' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], undefined) + assert.throws(() => directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'C1;x' }), /must be preserve, all, or a comma list/) + const servingImageDigest = `sha256:${'f'.repeat(64)}` + const harness = directorHarness({ servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' } }) + await deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'preserve' }, 'candidate-new', harness.operations) + assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'c1,c3') + const shrink = directorHarness({ + servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' }, + reserveCells: { 'https://relay-hash-uc.a.run.app': ['c3'], 'https://candidate-new---relay-hash-uc.a.run.app': ['c3'] } + }) + await assert.rejects( + deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'c1' }, 'candidate-new', shrink.operations), + /would drop reserve cells \(c3\)/ + ) +}) diff --git a/cloud/dev/scripts/operate-relay-cell-flags.mjs b/cloud/dev/scripts/operate-relay-cell-flags.mjs index 01f1ae56a10..1d962b4d2bd 100644 --- a/cloud/dev/scripts/operate-relay-cell-flags.mjs +++ b/cloud/dev/scripts/operate-relay-cell-flags.mjs @@ -16,18 +16,144 @@ export const READ_BACK_TIMEOUT_MS = 15_000 const READ_BACK_INTERVAL_MS = 1_000 const STORAGE_ROOT = 'https://storage.googleapis.com' +// Every switch a cell image may know, with what an absent key means on the cell. A cell +// resets a key missing from its object to this default, so a write must carry every key it +// does not mean to change: the tool reads the current object and changes only what is named. +const enumOf = (...options) => (value) => (options.includes(value) ? value : undefined) +const booleanValue = (value) => (value === 'true' ? true : value === 'false' ? false : undefined) +const numberIn = (min, max, integer) => (value) => { + const number = Number(value) + if (value === '' || !Number.isFinite(number) || number < min || number > max) return undefined + if (integer && !Number.isInteger(number)) return undefined + return number +} +export const CELL_FLAG_SPECS = { + readinessLocal: { parse: booleanValue, default: false }, + ticketCheck: { parse: enumOf('off', 'shadow', 'enforce'), default: 'off' }, + admitMode: { parse: enumOf('db', 'reserve'), default: 'db' }, + // Absent means the cell's region default. + intakePerSec: { + parse: (value) => { + const rate = numberIn(0, 1_000, false)(value) + return rate === 0 ? undefined : rate + } + }, + reserveDryRun: { parse: booleanValue, default: false }, + rejectionFence: { parse: booleanValue, default: true }, + // Absent keeps the cell's boot value. + readTimeoutMarginMs: { parse: numberIn(1_000, 60_000, true) }, + // Flip-back leases in flight; absent is a third of the cell's pool. + reregisterInFlight: { parse: numberIn(1, 16, true) } +} + +export const RESERVE_CONFIRMATION = 'RESERVE' +export const BREAK_GLASS_CONFIRMATION = 'BREAK GLASS' +// Longer than a director's cached read of the admit-mode table (5 s). +export const RESERVE_MODE_CACHE_WAIT_MS = 10_000 +// How long a flip back may take to lease every control the cell admitted from memory: its +// control count at a per-lane pace below the measured 0.38/s (170 ms trips), within the job. +export const REREGISTER_WAIT_MIN_MS = 8 * 60_000 +export const REREGISTER_WAIT_MAX_MS = 50 * 60_000 +const REREGISTER_CONTROLS_PER_LANE_PER_SEC = 0.3 +// The cell takes a third of its database pool, and caps the switch at the pool less two +// (host-session-registry.ts reregisterInFlightLimit). An image from before runtime-status +// reported the pool falls back to the deployed sizes (US 10, Asia 16). +const CELL_DATABASE_POOL_MAX = { 'asia-east2': 16 } +const CELL_DATABASE_POOL_MAX_DEFAULT = 10 + +export function reregisterLanes(runtime) { + const reported = runtime?.databasePoolMax + const pool = + Number.isInteger(reported) && reported > 0 + ? reported + : (CELL_DATABASE_POOL_MAX[runtime?.region] ?? CELL_DATABASE_POOL_MAX_DEFAULT) + const flag = runtime?.flagsApplied?.flags?.reregisterInFlight + if (flag === undefined) return Math.max(1, Math.floor(pool / 3)) + return Math.max(1, Math.min(Number(flag), pool - 2)) +} + +export function reregisterWaitMs(runtime) { + const controls = Number(runtime?.runtime?.controls ?? 0) + const lanes = reregisterLanes(runtime) + const estimate = (controls / (lanes * REREGISTER_CONTROLS_PER_LANE_PER_SEC)) * 1_000 + return Math.min(REREGISTER_WAIT_MAX_MS, Math.max(REREGISTER_WAIT_MIN_MS, Math.ceil(estimate))) +} + +// An image from before `supportedFlags` accepts the keys its applied flags list, and only +// ticketCheck off/shadow: anything else is dropped there, or voids the whole object. +function legacySupportedFlags(appliedFlags) { + return Object.fromEntries( + Object.entries(appliedFlags ?? {}).map(([key, value]) => [ + key, + key === 'ticketCheck' + ? { type: 'enum', values: ['off', 'shadow'] } + : { type: typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'number' : 'enum', values: [value] } + ]) + ) +} + +function accepts(spec, value) { + if (spec?.type === 'boolean') return typeof value === 'boolean' + if (spec?.type === 'enum') return spec.values.includes(value) + if (spec?.type === 'number') { + return ( + typeof value === 'number' && + (spec.min === undefined || value >= spec.min) && + (spec.max === undefined || value <= spec.max) && + (!spec.integer || Number.isInteger(value)) + ) + } + return false +} + +// The whole object as it will be written, not only the change: a bad key or value already in +// the file would void or be dropped on every later write too. +export function assertSupportedObject(flags, runtime) { + const supported = runtime.supportedFlags ?? legacySupportedFlags(runtime.flagsApplied?.flags) + for (const [key, value] of Object.entries(flags)) { + if (!accepts(supported[key], value)) { + throw new Error( + `the cell's image does not support ${key}=${JSON.stringify(value)}; roll the image first` + ) + } + } +} + +// `key=value,key=value`; `key=default` removes the key so the cell uses its default. +export function parseFlagChanges(text) { + const changes = {} + for (const part of (text ?? '').split(',').map((entry) => entry.trim()).filter(Boolean)) { + const separator = part.indexOf('=') + const key = separator < 0 ? part : part.slice(0, separator) + const raw = separator < 0 ? '' : part.slice(separator + 1).trim() + const spec = CELL_FLAG_SPECS[key] + if (!spec) throw new Error(`unknown switch ${key}`) + if (key in changes) throw new Error(`switch ${key} named twice`) + if (raw === 'default') { + changes[key] = null + continue + } + const value = spec.parse(raw) + if (value === undefined) throw new Error(`switch ${key} cannot be ${JSON.stringify(raw)}`) + changes[key] = value + } + if (Object.keys(changes).length === 0) throw new Error('set names no switch to change') + return changes +} + +// The switches as the cell will apply them: every known key, absent ones at their default. +function effectiveFlags(flags) { + return Object.fromEntries( + Object.entries(CELL_FLAG_SPECS).map(([key, spec]) => [key, flags?.[key] ?? spec.default]) + ) +} + export function parseCellFlagsRequest(values) { const mode = values.mode if (mode !== 'dry-run' && mode !== 'write') throw new Error('mode must be dry-run or write') const cellId = values['cell-id'] ?? '' if (!CELL_ID_PATTERN.test(cellId)) throw new Error('cell id is not a relay GCE cell id') - if (values['readiness-local'] !== 'true' && values['readiness-local'] !== 'false') { - throw new Error('readiness-local must be true or false') - } - const ticketCheck = values['ticket-check'] - if (ticketCheck !== 'off' && ticketCheck !== 'shadow') { - throw new Error('ticket-check must be off or shadow') - } + const changes = parseFlagChanges(values.set) // 0 means "no object yet": the write then creates it and refuses if one appeared. const expectedGeneration = values['expected-generation'] ?? '' if (!/^(0|[1-9][0-9]{0,18})$/.test(expectedGeneration)) { @@ -36,6 +162,29 @@ export function parseCellFlagsRequest(values) { if (mode === 'write' && values.confirmation !== `${WRITE_CONFIRMATION} ${cellId}`) { throw new Error(`write requires the confirmation "${WRITE_CONFIRMATION} ${cellId}"`) } + // Reserve moves this cell's hosts off the database: it is typed for this cell, every time. + if ( + mode === 'write' && + changes.admitMode === 'reserve' && + values['confirm-reserve'] !== `${RESERVE_CONFIRMATION} ${cellId}` + ) { + throw new Error(`admitMode=reserve requires --confirm-reserve "${RESERVE_CONFIRMATION} ${cellId}"`) + } + // Break glass: a dead or unreachable reserve cell cannot report db, so nothing would ever + // let the sweeps re-place its hosts. Only admitMode=db, typed for this cell. + const breakGlass = values['break-glass'] ?? '' + if (breakGlass !== '') { + if (breakGlass !== `${BREAK_GLASS_CONFIRMATION} ${cellId}`) { + throw new Error(`break glass requires --break-glass "${BREAK_GLASS_CONFIRMATION} ${cellId}"`) + } + if (mode !== 'write' || changes.admitMode !== null || Object.keys(changes).length !== 1) { + throw new Error('break glass only writes admitMode=default (db), on its own') + } + } + const directorOrigin = values['director-origin'] ?? '' + if (changes.admitMode !== undefined && !/^https:\/\/relay(-staging)?\.onorca\.dev$/.test(directorOrigin)) { + throw new Error('an admitMode change needs --director-origin, where Postgres records it') + } const projectId = values['project-id'] ?? '' if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId)) throw new Error('project id is invalid') const cellOrigin = values['cell-origin'] ?? '' @@ -49,16 +198,27 @@ export function parseCellFlagsRequest(values) { mode, cellId, cellOrigin, + directorOrigin, + breakGlass: breakGlass !== '', projectId, expectedGeneration, - object: { - v: 1, - cellId, - flags: { readinessLocal: values['readiness-local'] === 'true', ticketCheck } - } + changes } } +// Read-modify-write: the current object's switches with only the named ones changed. +export function desiredObject(request, current) { + if (current !== null && (current?.v !== 1 || current?.cellId !== request.cellId)) { + throw new Error('the current object is not this cell\'s v1 switch object; fix it by hand') + } + const flags = { ...(current?.flags ?? {}) } + for (const [key, value] of Object.entries(request.changes)) { + if (value === null) delete flags[key] + else flags[key] = value + } + return { v: 1, cellId: request.cellId, flags } +} + function objectUrl(request, suffix = '') { const bucket = `${request.projectId}-relay-control` return `${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${encodeURIComponent( @@ -101,7 +261,7 @@ async function readCurrentObject(fetchImpl, request, accessToken) { return { generation: String(generation), object: await media.json().catch(() => null) } } -async function writeObject(fetchImpl, request, accessToken, audit) { +async function writeObject(fetchImpl, request, object, accessToken, audit) { const boundary = `relay-cell-flags-${Date.now()}` const metadata = { name: `cells/${request.cellId}.json`, @@ -118,7 +278,7 @@ async function writeObject(fetchImpl, request, accessToken, audit) { `--${boundary}`, 'Content-Type: application/json', '', - JSON.stringify(request.object), + JSON.stringify(object), `--${boundary}--`, '' ].join('\r\n') @@ -134,21 +294,119 @@ async function writeObject(fetchImpl, request, accessToken, audit) { body } ) + // A 4xx means nothing was written; a 5xx or a lost reply may have been. if (response.status === 412) { await response.body?.cancel().catch(() => undefined) - throw new Error('the object changed since expected-generation; re-read and retry') + throw Object.assign(new Error('the object changed since expected-generation; re-read and retry'), { + notWritten: true + }) + } + if (!response.ok) { + throw Object.assign(new Error(`object write returned ${response.status}`), { + notWritten: response.status < 500 + }) } - if (!response.ok) throw new Error(`object write returned ${response.status}`) return String((await response.json()).generation) } -const sameFlags = (left, right) => - left?.readinessLocal === right.readinessLocal && left?.ticketCheck === right.ticketCheck +// The census's record (relay_cell_admit_modes, through a director). Without `admitMode` it only +// reads. Read back on every write: sweeps act on this, not on the cell's switch file. +async function directorAdmitMode(fetchImpl, request, idToken, wait, admitMode) { + const response = await fetchAdminOnceMore( + fetchImpl, + `${request.directorOrigin}/v1/admin/cell-admit-mode`, + { + method: 'POST', + headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId: request.cellId, ...(admitMode ? { admitMode } : {}) }) + }, + { wait, timeoutMs: 10_000, retryDelayMs: 1_000 } + ) + const body = await response.json().catch(() => null) + if (!response.ok) throw new Error(`director refused admitMode: ${response.status} ${body?.error ?? ''}`) + if (admitMode && body?.admitMode !== admitMode) { + throw new Error(`director recorded admitMode ${body?.admitMode}, not ${admitMode}`) + } + return body +} + +// Every known key, not just the ones this run changed: a cell that kept an old value of any +// switch has not applied this object. A key with no fixed default (the cell reports its own +// value) is compared only when the object names it. +export function sameFlags(applied, desired) { + const expected = effectiveFlags(desired) + const actual = effectiveFlags(applied) + return Object.entries(CELL_FLAG_SPECS).every( + ([key, spec]) => + (!('default' in spec) && desired?.[key] === undefined) || actual[key] === expected[key] + ) +} + +// Sweeps resume only once every control the cell admitted from memory holds a lease again. +async function waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log }) { + let waitMs + let deadline = now() + REREGISTER_WAIT_MIN_MS + while (now() < deadline) { + const runtime = await readRuntime(fetchImpl, request, idToken, sleep).catch(() => null) + // An image without the field never admitted from memory. + if (runtime && (runtime.admitModeEffective ?? 'db') === 'db') return + if (runtime && waitMs === undefined) { + waitMs = reregisterWaitMs(runtime) + deadline = now() + waitMs + log(JSON.stringify({ event: 'orca_relay_cell_reregistration_wait', cellId: request.cellId, waitMs })) + } + await sleep(5_000) + } + log(JSON.stringify({ event: 'orca_relay_cell_reregistration_still_running', cellId: request.cellId })) + throw new Error( + `cell still re-registering after ${waitMs ?? REREGISTER_WAIT_MIN_MS} ms. Its connections are safe and Postgres ` + + 'keeps reserve, so sweeps stay off the cell. Run again with --set admitMode=db, which records db once ' + + 'runtime-status reports admitModeEffective=db; to speed it up, name both in one write: ' + + '--set admitMode=db,reregisterInFlight=N' + ) +} + +// No cell read-back and no wait for its controls: the cell is not answering. The file says db +// so a cell that comes back boots in db, and Postgres says db so the sweeps re-place its hosts. +async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep }) { + const { accessToken, idToken, audit } = dependencies + // Only for a dead cell: one that answers, or still heartbeats, takes the ordinary flip back. + const answered = await fetchImpl(`${request.cellOrigin}/v1/admin/runtime-status`, { + method: 'POST', + headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1 }), + signal: AbortSignal.timeout(10_000) + }) + .then(async (response) => { + await response.body?.cancel().catch(() => undefined) + return response.ok + }) + .catch(() => false) + if (answered) throw new Error('the cell answers runtime-status: flip it back with --set admitMode=db, not break glass') + const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep) + if (recorded?.heartbeatFresh !== false) { + throw new Error( + `the director ${recorded?.heartbeatFresh ? 'still sees heartbeats from' : 'cannot say whether it hears'} ` + + 'the cell: break glass is only for a cell that has stopped' + ) + } + const current = await readCurrentObject(fetchImpl, request, accessToken) + if (current.generation !== request.expectedGeneration) { + throw new Error(`current generation is ${current.generation}, not expected ${request.expectedGeneration}`) + } + const object = desiredObject(request, current.object) + const generation = await writeObject(fetchImpl, request, object, accessToken, audit) + await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') + const result = { event: 'orca_relay_cell_admit_mode_break_glass', cellId: request.cellId, generation, audit } + log(JSON.stringify(result)) + return { ...result, written: true } +} export async function operateCellFlags(request, dependencies) { const { fetchImpl = fetch, accessToken, idToken, audit, log = console.log } = dependencies const now = dependencies.now ?? Date.now const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) + if (request.breakGlass) return await breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep }) const runtime = await readRuntime(fetchImpl, request, idToken, sleep) // Proves the origin is this cell, and that its image reads switch files at all. if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) { @@ -156,6 +414,20 @@ export async function operateCellFlags(request, dependencies) { } if (!runtime.flagsApplied) throw new Error('cell image has no flag channel (no flagsApplied)') const current = await readCurrentObject(fetchImpl, request, accessToken) + const object = desiredObject(request, current.object) + assertSupportedObject(object.flags, runtime) + // A tripped dead-man: any new generation that keeps reserve re-arms it, so the cell trips + // again and its re-registration restarts. Only a write that also says db is safe. + if ( + runtime.flagsApplied.flags?.admitMode === 'reserve' && + runtime.admitModeEffective === 'db' && + (object.flags.admitMode ?? 'db') === 'reserve' + ) { + throw new Error( + 'the cell tripped its dead-man (reserve configured, db effective); a write that keeps reserve re-arms it. ' + + 'Name admitMode=db in the same write, e.g. --set admitMode=db,reregisterInFlight=N' + ) + } const plan = { event: 'orca_relay_cell_flags_plan', mode: request.mode, @@ -163,7 +435,8 @@ export async function operateCellFlags(request, dependencies) { currentGeneration: current.generation, currentObject: current.object, appliedBefore: runtime.flagsApplied, - desired: request.object + changes: request.changes, + desired: object } // Before the generation check, so a dry run always shows the generation to pass next. log(JSON.stringify(plan)) @@ -172,8 +445,53 @@ export async function operateCellFlags(request, dependencies) { `current generation is ${current.generation}, not expected ${request.expectedGeneration}` ) } + // The cell applies every readable write; a generation it did not apply was voided there. A + // write of an object this image accepts whole (checked above) repairs it. + if (current.generation !== '0' && String(runtime.flagsApplied.generation) !== current.generation) { + log( + JSON.stringify({ + event: 'orca_relay_cell_flags_repairing_unapplied_object', + cellId: request.cellId, + appliedGeneration: runtime.flagsApplied.generation, + currentGeneration: current.generation + }) + ) + } + const admitModeChange = + request.changes.admitMode === undefined ? undefined : (request.changes.admitMode ?? 'db') + if (admitModeChange !== undefined) { + const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep) + log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded_before', ...recorded })) + } if (request.mode === 'dry-run') return { ...plan, written: false } - const generation = await writeObject(fetchImpl, request, accessToken, audit) + // Sweeps skip the cell before it starts admitting from memory, never after. + let generation + if (admitModeChange === 'reserve') { + const before = await directorAdmitMode(fetchImpl, request, idToken, sleep) + await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve') + // Every director's cached set expires (5 s) before the cell starts admitting from memory; + // the second write re-runs the open-flow check over anything started meanwhile. + await sleep(RESERVE_MODE_CACHE_WAIT_MS) + try { + await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve') + } catch (error) { + // A drain, migration or rehome opened during the wait: Postgres goes back as it was. + if (before?.admitMode === 'db') await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') + throw error + } + try { + generation = await writeObject(fetchImpl, request, object, accessToken, audit) + } catch (error) { + // The file was refused outright (412, 4xx): put Postgres back as it was. After a 5xx the + // object may be written, so Postgres keeps reserve (sweeps stay off: the safe side). + if (error?.notWritten && before?.admitMode === 'db') { + await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') + } + throw error + } + } else { + generation = await writeObject(fetchImpl, request, object, accessToken, audit) + } log(JSON.stringify({ event: 'orca_relay_cell_flags_written', cellId: request.cellId, generation })) const deadline = now() + READ_BACK_TIMEOUT_MS let last = runtime.flagsApplied @@ -185,9 +503,22 @@ export async function operateCellFlags(request, dependencies) { // The object is written; a failed poll only delays the read-back until the deadline. continue } - if (String(last?.generation) === generation && sameFlags(last.flags, request.object.flags)) { + if (String(last?.generation) !== generation) continue + // The image predates a switch this object names: it applied the rest and dropped that one. + if (Array.isArray(last.ignoredKeys) && last.ignoredKeys.length > 0) { + throw new Error( + `cell applied generation ${generation} but ignores ${last.ignoredKeys.join(', ')}; ` + + 'its image predates them, so flip the object back' + ) + } + if (sameFlags(last.flags, object.flags)) { const result = { ...plan, written: true, generation, applied: last } log(JSON.stringify({ ...result, event: 'orca_relay_cell_flags_applied_read_back' })) + if (admitModeChange === 'db') { + await waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log }) + await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') + log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded', cellId: request.cellId, admitMode: 'db' })) + } return result } } @@ -205,10 +536,12 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { 'cell-id', 'cell-origin', 'project-id', - 'readiness-local', - 'ticket-check', + 'set', 'expected-generation', - 'confirmation' + 'confirmation', + 'confirm-reserve', + 'break-glass', + 'director-origin' ].map((name) => [name, { type: 'string' }]) ) }) diff --git a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs index 23834755e33..3a5b565feff 100644 --- a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs +++ b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs @@ -1,36 +1,104 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { + assertSupportedObject, + desiredObject, operateCellFlags, parseCellFlagsRequest, + parseFlagChanges, READ_BACK_TIMEOUT_MS, + REREGISTER_WAIT_MAX_MS, + REREGISTER_WAIT_MIN_MS, + reregisterLanes, + reregisterWaitMs, WRITE_CONFIRMATION } from './operate-relay-cell-flags.mjs' const CELL = 'production-gce-c26' +const RESERVE = { 'director-origin': 'https://relay.onorca.dev', 'confirm-reserve': `RESERVE ${CELL}` } const values = (overrides = {}) => ({ mode: 'write', 'cell-id': CELL, 'cell-origin': 'https://c26.relay.onorca.dev', 'project-id': 'onorca-cloud', - 'readiness-local': 'true', - 'ticket-check': 'off', + set: 'readinessLocal=true', 'expected-generation': '0', confirmation: `${WRITE_CONFIRMATION} ${CELL}`, ...overrides }) // A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls. -function fakeGoogleAndCell({ stored = null, applyAfterPolls = 2, role = 'cell', cellId = CELL } = {}) { - const state = { stored, writes: [], polls: 0, applied: { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } } } +const SUPPORTED = { + readinessLocal: { type: 'boolean' }, + ticketCheck: { type: 'enum', values: ['off', 'shadow', 'enforce'] }, + admitMode: { type: 'enum', values: ['db', 'reserve'] }, + intakePerSec: { type: 'number', min: 0, max: 1_000 }, + reserveDryRun: { type: 'boolean' }, + rejectionFence: { type: 'boolean' }, + readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true }, + reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true } +} + +function fakeGoogleAndCell({ + stored = null, + applyAfterPolls = 2, + role = 'cell', + cellId = CELL, + supportedFlags = SUPPORTED, + // Polls that still report admitModeEffective=reserve after a flip back. + reregisteringPolls = 0, + controls, + // Forces admitModeEffective, as a tripped dead-man reports it. + effectiveOverride, + heartbeatFresh = false, + // The director refuses this numbered reserve post (1-based) with this error. + refuseReservePost +} = {}) { + const state = { + stored, + writes: [], + polls: 0, + // A cell that has applied whatever object is already there. + applied: stored + ? { generation: Number(stored.generation), flags: stored.object.flags } + : { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } }, + directorAdmitMode: 'db', + reservePosts: 0, + order: [], + reregisteringPolls + } const fetchImpl = async (url, init = {}) => { const target = new URL(url) + if (target.pathname === '/v1/admin/cell-admit-mode') { + const body = JSON.parse(String(init.body)) + if (body.admitMode === 'reserve') { + state.reservePosts += 1 + if (refuseReservePost?.post === state.reservePosts) { + return Response.json({ error: refuseReservePost.error }, { status: 409 }) + } + } + if (body.admitMode) { + state.directorAdmitMode = body.admitMode + state.order.push(`director:${body.admitMode}`) + } + return Response.json({ v: 1, cellId: body.cellId, admitMode: state.directorAdmitMode, updatedAt: 1, heartbeatFresh }) + } if (target.pathname === '/v1/admin/runtime-status') { state.polls += 1 if (state.stored && state.polls > applyAfterPolls) { state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags } } - return Response.json({ v: 1, role, cellId, flagsApplied: state.applied }) + const reserveApplied = state.applied.flags.admitMode === 'reserve' + const effective = reserveApplied || (state.writes.length > 0 && state.reregisteringPolls-- > 0) ? 'reserve' : 'db' + return Response.json({ + v: 1, + role, + cellId, + flagsApplied: state.applied, + admitModeEffective: effectiveOverride ?? effective, + ...(controls === undefined ? {} : { runtime: { controls } }), + ...(supportedFlags ? { supportedFlags } : {}) + }) } if (target.pathname.startsWith('/upload/')) { const expected = target.searchParams.get('ifGenerationMatch') @@ -41,6 +109,7 @@ function fakeGoogleAndCell({ stored = null, applyAfterPolls = 2, role = 'cell', const generation = String(Number(state.stored?.generation ?? 1000) + 1) state.stored = { generation, object } state.writes.push({ generation, object, metadata }) + state.order.push(`file:${object.flags.admitMode ?? 'db'}`) return Response.json({ generation }) } if (!state.stored) return new Response(null, { status: 404 }) @@ -73,7 +142,11 @@ const run = (request, fake, extra = {}) => { test('refuses malformed requests before touching anything', () => { assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/) assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/) - assert.throws(() => parseCellFlagsRequest(values({ 'ticket-check': 'enforce' })), /ticket-check/) + assert.throws(() => parseCellFlagsRequest(values({ set: 'ticketCheck=always' })), /ticketCheck/) + assert.throws(() => parseCellFlagsRequest(values({ set: 'placerMode=on' })), /unknown switch/) + assert.throws(() => parseCellFlagsRequest(values({ set: '' })), /names no switch/) + assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db,admitMode=reserve' })), /twice/) + assert.throws(() => parseCellFlagsRequest(values({ set: 'readTimeoutMarginMs=500' })), /cannot be/) assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/) assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/) assert.throws( @@ -92,7 +165,7 @@ test('a dry run reports the change and writes nothing', async () => { assert.equal(outcome.written, false) assert.equal(fake.state.writes.length, 0) assert.equal(lines[0].event, 'orca_relay_cell_flags_plan') - assert.deepEqual(lines[0].desired.flags, { readinessLocal: true, ticketCheck: 'off' }) + assert.deepEqual(lines[0].desired.flags, { readinessLocal: true }) }) test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => { @@ -101,12 +174,12 @@ test('writes with compare-and-swap, records who wrote it, and waits for the cell const outcome = await result assert.equal(outcome.written, true) assert.equal(outcome.generation, '1001') - assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true, ticketCheck: 'off' } }) + assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true } }) assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' }) assert.deepEqual(fake.state.writes[0].object, { v: 1, cellId: CELL, - flags: { readinessLocal: true, ticketCheck: 'off' } + flags: { readinessLocal: true } }) assert.deepEqual(lines.map((line) => line.event), [ 'orca_relay_cell_flags_plan', @@ -115,7 +188,7 @@ test('writes with compare-and-swap, records who wrote it, and waits for the cell ]) // Flip back: the next write must name the generation just written. const back = run( - parseCellFlagsRequest(values({ 'readiness-local': 'false', 'expected-generation': '1001' })), + parseCellFlagsRequest(values({ set: 'readinessLocal=false', 'expected-generation': '1001' })), fake ) assert.equal((await back.result).generation, '1002') @@ -182,3 +255,269 @@ test('fails loudly when the cell does not apply the write within the read-back w await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/) assert.equal(fake.state.writes.length, 1) }) + +test('changes only the named switches and keeps every other key the object already has', () => { + const current = { + v: 1, + cellId: CELL, + flags: { readinessLocal: true, ticketCheck: 'shadow', readTimeoutMarginMs: 5_000 } + } + const request = parseCellFlagsRequest( + values({ set: 'admitMode=reserve, intakePerSec=40, readTimeoutMarginMs=default', ...RESERVE }) + ) + assert.deepEqual(desiredObject(request, current), { + v: 1, + cellId: CELL, + flags: { readinessLocal: true, ticketCheck: 'shadow', admitMode: 'reserve', intakePerSec: 40 } + }) + assert.deepEqual(desiredObject(request, null).flags, { admitMode: 'reserve', intakePerSec: 40 }) + assert.throws(() => desiredObject(request, { v: 1, cellId: 'production-gce-c27', flags: {} }), /by hand/) + assert.deepEqual(parseFlagChanges('rejectionFence=false'), { rejectionFence: false }) + assert.deepEqual(parseFlagChanges('reregisterInFlight=8'), { reregisterInFlight: 8 }) + assert.throws(() => parseFlagChanges('reregisterInFlight=32'), /cannot be/) +}) + +test('a write keeps the switches it did not name, end to end', async () => { + const fake = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } } + }) + const outcome = await run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result + assert.equal(outcome.written, true) + assert.deepEqual(fake.state.writes[0].object.flags, { ticketCheck: 'shadow', readinessLocal: true }) +}) + +test('the read-back compares every switch, not only the one this run changed', async () => { + const fake = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } } + }) + const inner = fake.fetchImpl + fake.fetchImpl = async (url, init) => { + const response = await inner(url, init) + if (new URL(url).pathname !== '/v1/admin/runtime-status') return response + const body = await response.json() + // The cell reports the new generation but still runs the old ticketCheck. + if (body.flagsApplied.generation === 8) body.flagsApplied.flags = { readinessLocal: true } + return Response.json(body) + } + await assert.rejects( + run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result, + /did not apply generation 8/ + ) +}) + +test('a switch the cell reports on its own is compared only when the object names it', async () => { + const fake = fakeGoogleAndCell() + const inner = fake.fetchImpl + fake.fetchImpl = async (url, init) => { + const response = await inner(url, init) + if (new URL(url).pathname !== '/v1/admin/runtime-status') return response + const body = await response.json() + body.flagsApplied.flags = { ...body.flagsApplied.flags, intakePerSec: 50, readTimeoutMarginMs: 10_000 } + return Response.json(body) + } + assert.equal((await run(parseCellFlagsRequest(values()), fake).result).written, true) +}) + +test('fails the read-back when the cell ignores a switch the object names', async () => { + const fake = fakeGoogleAndCell() + const inner = fake.fetchImpl + fake.fetchImpl = async (url, init) => { + const response = await inner(url, init) + if (new URL(url).pathname !== '/v1/admin/runtime-status') return response + const body = await response.json() + if (body.flagsApplied.generation === 1001) body.flagsApplied.ignoredKeys = ['admitMode'] + return Response.json(body) + } + await assert.rejects( + run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), fake).result, + /ignores admitMode/ + ) +}) + +test('refuses a switch the cell image does not support, before writing anything', async () => { + const old = fakeGoogleAndCell({ supportedFlags: null }) + await assert.rejects( + run(parseCellFlagsRequest(values({ set: 'ticketCheck=enforce' })), old).result, + /does not support ticketCheck="enforce"/ + ) + await assert.rejects( + run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), old).result, + /does not support admitMode/ + ) + assert.equal(old.state.writes.length, 0) + // An image from before supportedFlags still takes the two switches it always had. + assert.equal((await run(parseCellFlagsRequest(values()), old).result).written, true) + assert.throws( + () => assertSupportedObject({ readTimeoutMarginMs: 1_500.5 }, { supportedFlags: SUPPORTED }), + /readTimeoutMarginMs/ + ) + assertSupportedObject({ ticketCheck: 'enforce' }, { supportedFlags: SUPPORTED }) +}) + +test('refuses to carry a bad value already in the file', async () => { + const bad = fakeGoogleAndCell({ + supportedFlags: null, + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'enforce' } } } + }) + // The old image voided generation 7 and still runs generation 0. + bad.state.applied = { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } } + await assert.rejects( + run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), bad).result, + /does not support ticketCheck="enforce"/ + ) + assert.equal(bad.state.writes.length, 0) +}) + +const DIRECTOR = { 'director-origin': RESERVE['director-origin'] } + +test('admitMode=reserve needs a typed confirmation, and Postgres records reserve before the file', async () => { + assert.throws( + () => parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=reserve' })), + /--confirm-reserve "RESERVE production-gce-c26"/ + ) + assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db' })), /--director-origin/) + const fake = fakeGoogleAndCell() + const request = parseCellFlagsRequest( + values({ ...DIRECTOR, set: 'admitMode=reserve', 'confirm-reserve': `RESERVE ${CELL}` }) + ) + assert.equal((await run(request, fake).result).written, true) + // Recorded, re-checked after every director's cache has expired, then the file. + assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'file:reserve']) +}) + +test('a refused file write after Postgres says reserve puts Postgres back to db', async () => { + const fake = fakeGoogleAndCell() + const inner = fake.fetchImpl + fake.fetchImpl = async (url, init) => { + if (new URL(url).pathname.startsWith('/upload/')) return new Response(null, { status: 403 }) + return await inner(url, init) + } + const request = parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })) + await assert.rejects(run(request, fake).result, /returned 403/) + assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'director:db']) +}) + +test('repairs an object the cell did not apply when the whole new object is valid', async () => { + const voided = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } } + }) + voided.state.applied = { generation: 3, flags: { readinessLocal: false } } + const { result, lines } = run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), voided) + assert.equal((await result).written, true) + assert.ok(lines.some((line) => line.event === 'orca_relay_cell_flags_repairing_unapplied_object')) +}) + +test('a flip back records db in Postgres only after the cell has leased every control', async () => { + const fake = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }, + reregisteringPolls: 3 + }) + fake.state.directorAdmitMode = 'reserve' + const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' })) + assert.equal((await run(request, fake).result).written, true) + assert.deepEqual(fake.state.order, ['file:db', 'director:db']) + assert.equal(fake.state.reregisteringPolls, -1) +}) + +test('the flip-back wait scales with the control count and the switch-file pace', () => { + assert.equal(reregisterWaitMs({ runtime: { controls: 100 } }), REREGISTER_WAIT_MIN_MS) + // 1,500 controls at the default 3 lanes: 1,667 s. + assert.equal(reregisterWaitMs({ runtime: { controls: 1_500 } }), 1_666_667) + assert.equal( + reregisterWaitMs({ runtime: { controls: 1_500 }, flagsApplied: { flags: { reregisterInFlight: 10 } } }), + // The cell caps 10 at its US pool of 10 less two. + 625_000 + ) + assert.equal(reregisterWaitMs({ runtime: { controls: 3_000 } }), REREGISTER_WAIT_MAX_MS) +}) + +test('the wait counts the lanes the cell actually uses', () => { + assert.equal(reregisterLanes({ region: 'us-central1' }), 3) + assert.equal(reregisterLanes({ region: 'asia-east2' }), 5) + assert.equal(reregisterLanes({ region: 'us-central1', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 8) + assert.equal(reregisterLanes({ region: 'asia-east2', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 14) + // The pool the cell reports wins over the deployed-size table. + assert.equal(reregisterLanes({ region: 'us-central1', databasePoolMax: 22 }), 7) + assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 12, flagsApplied: { flags: { reregisterInFlight: 16 } } }), 10) + assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 'x' }), 5) +}) + +test('a tripped cell takes no write that keeps reserve, only one that also says db', async () => { + const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } } + const tripped = () => fakeGoogleAndCell({ stored, effectiveOverride: 'db' }) + for (const set of ['readinessLocal=true', 'reregisterInFlight=4']) { + const fake = tripped() + const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' })) + await assert.rejects(run(request, fake).result, /tripped its dead-man[\s\S]*admitMode=db,reregisterInFlight=N/) + assert.deepEqual(fake.state.writes, []) + } + const fake = tripped() + fake.state.directorAdmitMode = 'reserve' + const request = parseCellFlagsRequest( + values({ ...DIRECTOR, set: 'admitMode=db,reregisterInFlight=4', 'expected-generation': '7' }) + ) + assert.equal((await run(request, fake).result).written, true) + assert.deepEqual(fake.state.writes[0].object.flags, { admitMode: 'db', reregisterInFlight: 4 }) +}) + +test('a migration opened during the cache wait puts Postgres back to db', async () => { + const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } }) + const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'admitMode=reserve' })) + await assert.rejects(run(request, fake).result, /cell_has_open_migration/) + assert.deepEqual(fake.state.order, ['director:reserve', 'director:db']) + assert.deepEqual(fake.state.writes, []) +}) + +test('break glass refuses a cell that answers or that the director still hears', async () => { + const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7', 'break-glass': `BREAK GLASS ${CELL}` } + const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } } + const answering = fakeGoogleAndCell({ stored }) + await assert.rejects(run(parseCellFlagsRequest(values(glass)), answering).result, /answers runtime-status/) + const heard = fakeGoogleAndCell({ stored, heartbeatFresh: true }) + const inner = heard.fetchImpl + heard.fetchImpl = async (url, init) => { + if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused') + return await inner(url, init) + } + await assert.rejects(run(parseCellFlagsRequest(values(glass)), heard).result, /still sees heartbeats/) + assert.deepEqual(heard.state.writes, []) + assert.deepEqual(answering.state.writes, []) +}) + +test('a flip back that outlasts its wait keeps Postgres on reserve and says how to finish', async () => { + const fake = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }, + reregisteringPolls: Number.POSITIVE_INFINITY, + controls: 1_500 + }) + fake.state.directorAdmitMode = 'reserve' + const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' })) + const { lines, result } = run(request, fake) + await assert.rejects(result, /after 1666667 ms[\s\S]*Run again with --set admitMode=db[\s\S]*--set admitMode=db,reregisterInFlight=N/) + assert.deepEqual(fake.state.order, ['file:db']) + assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667) +}) + +test('break glass records db for a cell that cannot answer, typed for that cell only', async () => { + const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' } + assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/) + assert.throws( + () => parseCellFlagsRequest(values({ ...glass, set: 'admitMode=default,readinessLocal=true', 'break-glass': `BREAK GLASS ${CELL}` })), + /on its own/ + ) + const fake = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true } } } + }) + fake.state.directorAdmitMode = 'reserve' + const inner = fake.fetchImpl + fake.fetchImpl = async (url, init) => { + if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused') + return await inner(url, init) + } + const { result, lines } = run(parseCellFlagsRequest(values({ ...glass, 'break-glass': `BREAK GLASS ${CELL}` })), fake) + assert.equal((await result).written, true) + assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: true }) + assert.deepEqual(fake.state.order, ['file:db', 'director:db']) + assert.equal(lines.at(-1).event, 'orca_relay_cell_admit_mode_break_glass') +}) + diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs index 796d0e9d91a..b8e368f6e1b 100644 --- a/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs +++ b/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs @@ -53,7 +53,9 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = { try { service = run(gcloudArguments('services', input)) } catch (error) { - if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' } + if (error?.code === 'NOT_FOUND') { + return { version: input.bootstrap_version, cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' } + } throw error } const serving = (service.status?.traffic ?? []).filter( @@ -77,12 +79,21 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = { throw new Error('serving region correction cohort is invalid') } const cohort_percent = cohortSettings[0]?.value ?? '0' + // Set by the director deploy (placement) or by hand (shadow cells), never by Terraform: carried + // so an apply keeps them, and empty when the serving revision has none. + const reserve_placement = servingPlainSetting(revision, 'ORCA_RELAY_RESERVE_PLACEMENT', /^(?:off|dry-run|on)$/) + const shadow_seat_feed_cells = servingPlainSetting( + revision, + 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', + /^(?:all|[A-Za-z0-9_-]+(?:, ?[A-Za-z0-9_-]+)*)$/ + ) + const carried = { cohort_percent, reserve_placement, shadow_seat_feed_cells } const references = (revision.spec?.containers ?? []).flatMap((container) => (container.env ?? []).filter( (environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' ) ) - if (references.length === 0) return { version: input.bootstrap_version, cohort_percent } + if (references.length === 0) return { version: input.bootstrap_version, ...carried } const reference = normalizeSecretReference(references[0]) if ( references.length !== 1 || @@ -91,7 +102,20 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = { ) { throw new Error('serving regional placement secret reference is invalid') } - return { version: reference.version, cohort_percent } + return { version: reference.version, ...carried } +} + +// One plain value, or none; anything else (a secret, a duplicate, a malformed value) fails the +// plan rather than letting an apply drop or reset the setting. +function servingPlainSetting(revision, name, valid) { + const settings = (revision.spec?.containers ?? []).flatMap((container) => + (container.env ?? []).filter((environment) => environment.name === name) + ) + if (settings.length === 0) return '' + if (settings.length > 1 || typeof settings[0].value !== 'string' || !valid.test(settings[0].value)) { + throw new Error(`serving ${name} is invalid`) + } + return settings[0].value } // Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs index 9c49d4127a5..8fd0c3a551e 100644 --- a/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs +++ b/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs @@ -1,5 +1,7 @@ import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' import test from 'node:test' +import { fileURLToPath } from 'node:url' import { classifyRelayServiceDescribeFailure, readRelayServingRegionalPlacementVersion @@ -67,7 +69,7 @@ test('reads the exact version from the sole traffic-serving revision', () => { } }) - assert.deepEqual(result, { version: '11', cohort_percent: '0' }) + assert.deepEqual(result, { version: '11', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) assert.equal(calls[1][3], 'relay-serving') }) @@ -78,7 +80,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource ]) { assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1') - }), { version: '1', cohort_percent: '0' }) + }), { version: '1', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) } }) @@ -100,12 +102,12 @@ test('falls back only when the service or setting is absent', () => { notFound.code = 'NOT_FOUND' assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { run: () => { throw notFound } - }), { version: '7', cohort_percent: '0' }) + }), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { run: (args) => args[1] === 'services' ? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } : { spec: { containers: [{ env: [] }] } } - }), { version: '7', cohort_percent: '0' }) + }), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) }) test('classifies real absent-service stderr without weakening revision failures', () => { @@ -144,7 +146,7 @@ test('preserves the serving cohort including explicit disable across later Terra servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value }) assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { run: (args) => args[1] === 'services' ? serving() : servingRevision - }), { version: '11', cohort_percent: value }) + }), { version: '11', cohort_percent: value, reserve_placement: '', shadow_seat_feed_cells: '' }) } }) @@ -164,3 +166,44 @@ test('fails closed on malformed, secret-backed or duplicate cohorts rather than }), /cohort is invalid/) } }) + +test('carries the deploy-owned placement and shadow cells, so an apply keeps them', () => { + const servingRevision = revision() + servingRevision.spec.containers[0].env.push( + { name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' }, + { name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'production-gce-c1,production-gce-c2' } + ) + assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' ? serving() : servingRevision + }), { + version: '11', + cohort_percent: '0', + reserve_placement: 'on', + shadow_seat_feed_cells: 'production-gce-c1,production-gce-c2' + }) +}) + +test('fails closed on a malformed, secret-backed or duplicate placement or shadow-cell setting', () => { + const cases = [ + [{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'yes' }], + [{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' }, { name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'off' }], + [{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', valueFrom: { secretKeyRef: { name: 'x', key: '1' } } }], + [{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'c1;c2' }], + [{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: '' }] + ] + for (const settings of cases) { + const servingRevision = revision() + servingRevision.spec.containers[0].env.push(...settings) + assert.throws(() => readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' ? serving() : servingRevision + }), /is invalid/) + } +}) + +test('Terraform sets the carried settings only when the serving revision has them', () => { + const terraform = readFileSync(fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), 'utf8') + const block = /dynamic "env" \{([\s\S]*?)\n {6}\}/.exec(terraform)?.[1] ?? '' + assert.match(block, /ORCA_RELAY_RESERVE_PLACEMENT *= data\.external\.relay_serving_regional_placement_version\.result\.reserve_placement/) + assert.match(block, /ORCA_RELAY_SHADOW_SEAT_FEED_CELLS *= data\.external\.relay_serving_regional_placement_version\.result\.shadow_seat_feed_cells/) + assert.match(block, /if value != ""/) +}) diff --git a/cloud/dev/scripts/relay-deploy-workflow-argv.test.mjs b/cloud/dev/scripts/relay-deploy-workflow-argv.test.mjs new file mode 100644 index 00000000000..a4861625084 --- /dev/null +++ b/cloud/dev/scripts/relay-deploy-workflow-argv.test.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict' +import { readdirSync, readFileSync } from 'node:fs' +import test from 'node:test' +import { parseArguments as parseBlueGreen } from './deploy-relay-blue-green.mjs' +import { parseArguments as parseGceCandidate } from './deploy-relay-gce-candidate.mjs' +import { RELAY_WORKFLOW_DIRECTORY, readRelayWorkflow } from './relay-repository.mjs' + +// Every workflow's real argv, literals as written: a flag combination the parser refuses +// (staging's lone --admin-audience once was one) fails here instead of on a release. + +const PROJECT = 'orca-test-project' +const DIGEST = `sha256:${'a'.repeat(64)}` + +// Shell-expanded values only; anything a workflow writes literally is parsed as written. +const SAMPLES = { + project: PROJECT, + image: `relay@${DIGEST}`, + 'predecessor-image-digest': DIGEST, + 'director-cells-json': JSON.stringify([ + { id: 'cell-a', url: 'https://cell-a.example.test', region: 'us-central1', capacityRequests: 10, initiallyEnabled: true } + ]), + 'regional-placement-secret-version': '3', + 'min-instances': '1', + 'max-instances': '5', + 'prune-revisions': 'false', + 'bootstrap-runtime-identity': 'false', + 'expected-rehome-generation': '4', + 'region-correction-cohort-percent': '10', + 'rehome-audience': 'https://relay.example.test/v1/admin/host-drain', + 'director-origin': 'https://relay.example.test', + 'admin-audience': 'https://relay.example.test/v1/admin/drain', + 'reserve-placement': 'preserve', + 'shadow-seat-feed-cells': 'preserve', + mode: 'audit' +} + +function sample(flag) { + if (flag in SAMPLES) return SAMPLES[flag] + if (flag.endsWith('service-account')) return `${flag.slice(0, 20).replace(/-$/, '')}@${PROJECT}.iam.gserviceaccount.com` + return `sample-${flag}` +} + +function invocations(source, script) { + const lines = source.split('\n') + const found = [] + for (let index = 0; index < lines.length; index++) { + if (!lines[index].includes(`dev/scripts/${script}`)) continue + const argv = [] + let line = lines[index] + while (line.trimEnd().endsWith('\\')) { + line = lines[++index] + const match = /^\s*(--[a-z0-9-]+)\s+(.+?)\s*\\?\s*$/.exec(line) + if (!match) throw new Error(`unparsed ${script} argument line: ${line.trim()}`) + const [, flag, raw] = match + // A trailing `)"` closes the command substitution the invocation sits in. + const value = raw.replace(/\)"$/, '').replace(/^"(.*)"$/, '$1') + argv.push(flag, value.includes('$') ? sample(flag.slice(2)) : value) + } + found.push(argv) + } + return found +} + +const SCRIPTS = [ + ['deploy-relay-blue-green.mjs', parseBlueGreen], + ['deploy-relay-gce-candidate.mjs', parseGceCandidate] +] + +for (const [script, parse] of SCRIPTS) { + test(`every workflow's ${script} argv parses`, () => { + let count = 0 + for (const file of readdirSync(RELAY_WORKFLOW_DIRECTORY).filter((name) => name.endsWith('.yml'))) { + for (const argv of invocations(readFileSync(new URL(file, RELAY_WORKFLOW_DIRECTORY), 'utf8'), script)) { + count++ + assert.doesNotThrow(() => parse(argv), `${file}: ${argv.join(' ')}`) + } + } + assert.ok(count > 0, `no workflow runs ${script}`) + }) +} + +test('the production and staging director workflows are among the parsed', () => { + for (const name of ['deploy-relay-production-director.yml', 'deploy-relay-staging.yml']) { + assert.equal(invocations(readRelayWorkflow(name), 'deploy-relay-blue-green.mjs').length, 1, name) + } +}) diff --git a/cloud/dev/scripts/relay-director-deploy-plan.mjs b/cloud/dev/scripts/relay-director-deploy-plan.mjs index aaeb82bddc9..5f45de5afb5 100644 --- a/cloud/dev/scripts/relay-director-deploy-plan.mjs +++ b/cloud/dev/scripts/relay-director-deploy-plan.mjs @@ -211,6 +211,8 @@ export function directorDeployInputs({ imageDigest, predecessorDigest, rehomeGen 'image-digest': imageDigest, 'regional-placement-mode': 'preserve', 'region-correction-cohort-percent': 'preserve', + 'reserve-placement': 'preserve', + 'shadow-seat-feed-cells': 'preserve', 'prune-incompatible-revisions': 'false', 'expected-rehome-generation': String(rehomeGeneration), 'bootstrap-runtime-identity': 'false', diff --git a/cloud/infra/terraform/relay.tf b/cloud/infra/terraform/relay.tf index 5df7372ff07..217f92cfd47 100644 --- a/cloud/infra/terraform/relay.tf +++ b/cloud/infra/terraform/relay.tf @@ -174,6 +174,22 @@ resource "google_cloud_run_v2_service" "relay" { value = data.external.relay_serving_regional_placement_version.result.cohort_percent } + # The director deploy owns these (shadow cells are set by hand): an apply keeps whatever the + # serving revision has, and adds nothing when it has none. + dynamic "env" { + for_each = { + for name, value in { + ORCA_RELAY_RESERVE_PLACEMENT = data.external.relay_serving_regional_placement_version.result.reserve_placement + ORCA_RELAY_SHADOW_SEAT_FEED_CELLS = data.external.relay_serving_regional_placement_version.result.shadow_seat_feed_cells + } : name => value if value != "" + } + + content { + name = env.key + value = env.value + } + } + ports { container_port = 8080 }