diff --git a/.github/workflows/cloud-operate-relay-production-cell-flags.yml b/.github/workflows/cloud-operate-relay-production-cell-flags.yml new file mode 100644 index 00000000000..74576e0f6c7 --- /dev/null +++ b/.github/workflows/cloud-operate-relay-production-cell-flags.yml @@ -0,0 +1,111 @@ +name: Operate Relay Production Cell Flags + +# Writes one cell's runtime switch object and waits for the cell to report it applied. +# One cell per run; never fleet-wide. dry-run writes nothing. +on: + workflow_dispatch: + inputs: + mode: + description: dry-run prints the change; write applies it + required: true + default: dry-run + type: choice + options: [dry-run, write] + cell-id: + description: Exact cell id, e.g. production-gce-c26 + required: true + type: string + readiness-local: + description: /ready answers from the cell alone (local readiness) + required: true + default: 'false' + type: choice + options: ['false', 'true'] + ticket-check: + description: Shadow-check the director's assignment lease on host hellos + required: true + default: 'off' + type: choice + options: ['off', 'shadow'] + expected-generation: + description: Exact current object generation from a dry run, or 0 if none exists + required: true + type: string + confirmation: + description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS " + required: false + type: string + +permissions: + contents: read + id-token: write + +# One write at a time across the fleet keeps the audit trail in order. +concurrency: + group: relay-production-cell-flags + cancel-in-progress: false + +defaults: + run: + # Explicit bash runs with pipefail, so the script's exit code survives the tee. + shell: bash + working-directory: cloud + +jobs: + operate: + if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: production + env: + MODE: ${{ inputs.mode }} + CELL_ID: ${{ inputs.cell-id }} + READINESS_LOCAL: ${{ inputs.readiness-local }} + TICKET_CHECK: ${{ inputs.ticket-check }} + EXPECTED_GENERATION: ${{ inputs.expected-generation }} + CONFIRMATION: ${{ inputs.confirmation }} + GCP_PROJECT_ID: onorca-cloud + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - id: google-auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} + token_format: id_token + id_token_audience: https://relay.onorca.dev/v1/admin/drain + id_token_include_email: true + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Write or preview the cell's switch object + env: + ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} + run: | + [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]] + ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)" + echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}" + export ORCA_RELAY_CONTROL_ACCESS_TOKEN + node dev/scripts/operate-relay-cell-flags.mjs \ + --mode "${MODE}" --cell-id "${CELL_ID}" \ + --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ + --project-id "${GCP_PROJECT_ID}" \ + --readiness-local "${READINESS_LOCAL}" --ticket-check "${TICKET_CHECK}" \ + --expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \ + | tee "${RUNNER_TEMP}/relay-cell-flags.jsonl" + + - name: Audit summary + if: ${{ always() }} + run: | + { + echo "### Relay cell flags: ${MODE} ${CELL_ID}" + echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" + echo '```json' + cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)' + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/cloud/dev/scripts/operate-relay-cell-flags.mjs b/cloud/dev/scripts/operate-relay-cell-flags.mjs new file mode 100644 index 00000000000..01f1ae56a10 --- /dev/null +++ b/cloud/dev/scripts/operate-relay-cell-flags.mjs @@ -0,0 +1,240 @@ +#!/usr/bin/env node +// Writes one cell's runtime switch object (cells/.json in -relay-control) and +// reads back what the cell applied. One cell per run by construction: the object is per cell. +// +// dry-run: checks the cell and the current generation, prints the change, writes nothing. +// write: the same checks, then a compare-and-swap write (ifGenerationMatch), then waits for +// the cell's runtime status to report the new generation applied. +import { parseArgs } from 'node:util' +import { pathToFileURL } from 'node:url' +import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' + +export const CELL_ID_PATTERN = /^(production|staging)-gce-c[0-9]{1,3}$/ +export const WRITE_CONFIRMATION = 'WRITE_RELAY_CELL_FLAGS' +// The cell polls every 5 s (+20% jitter); 15 s leaves two polls of slack. +export const READ_BACK_TIMEOUT_MS = 15_000 +const READ_BACK_INTERVAL_MS = 1_000 +const STORAGE_ROOT = 'https://storage.googleapis.com' + +export function parseCellFlagsRequest(values) { + const mode = values.mode + if (mode !== 'dry-run' && mode !== 'write') throw new Error('mode must be dry-run or write') + const cellId = values['cell-id'] ?? '' + if (!CELL_ID_PATTERN.test(cellId)) throw new Error('cell id is not a relay GCE cell id') + if (values['readiness-local'] !== 'true' && values['readiness-local'] !== 'false') { + throw new Error('readiness-local must be true or false') + } + const ticketCheck = values['ticket-check'] + if (ticketCheck !== 'off' && ticketCheck !== 'shadow') { + throw new Error('ticket-check must be off or shadow') + } + // 0 means "no object yet": the write then creates it and refuses if one appeared. + const expectedGeneration = values['expected-generation'] ?? '' + if (!/^(0|[1-9][0-9]{0,18})$/.test(expectedGeneration)) { + throw new Error('expected-generation must be 0 or the exact current object generation') + } + if (mode === 'write' && values.confirmation !== `${WRITE_CONFIRMATION} ${cellId}`) { + throw new Error(`write requires the confirmation "${WRITE_CONFIRMATION} ${cellId}"`) + } + const projectId = values['project-id'] ?? '' + if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId)) throw new Error('project id is invalid') + const cellOrigin = values['cell-origin'] ?? '' + if (!/^https:\/\/c[0-9]{1,3}\.relay(-staging)?\.onorca\.dev$/.test(cellOrigin)) { + throw new Error('cell origin is not a relay cell origin') + } + if (new URL(cellOrigin).hostname.split('.')[0] !== cellId.replace(/^[a-z]+-gce-/, '')) { + throw new Error('cell origin names another cell') + } + return { + mode, + cellId, + cellOrigin, + projectId, + expectedGeneration, + object: { + v: 1, + cellId, + flags: { readinessLocal: values['readiness-local'] === 'true', ticketCheck } + } + } +} + +function objectUrl(request, suffix = '') { + const bucket = `${request.projectId}-relay-control` + return `${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${encodeURIComponent( + `cells/${request.cellId}.json` + )}${suffix}` +} + +// One transient 5xx or network failure is retried once, as every relay admin step does. +async function readRuntime(fetchImpl, request, idToken, wait) { + const response = await fetchAdminOnceMore( + fetchImpl, + `${request.cellOrigin}/v1/admin/runtime-status`, + { + method: 'POST', + headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1 }) + }, + { wait, timeoutMs: 10_000, retryDelayMs: 1_000 } + ) + if (!response.ok) { + await response.body?.cancel().catch(() => undefined) + throw new Error(`cell runtime status returned ${response.status}`) + } + return await response.json() +} + +async function readCurrentObject(fetchImpl, request, accessToken) { + const headers = { authorization: `Bearer ${accessToken}` } + const metadata = await fetchImpl(objectUrl(request), { headers }) + if (metadata.status === 404) { + await metadata.body?.cancel().catch(() => undefined) + return { generation: '0', object: null } + } + if (!metadata.ok) throw new Error(`object metadata read returned ${metadata.status}`) + const { generation } = await metadata.json() + const media = await fetchImpl(objectUrl(request, `?alt=media&ifGenerationMatch=${generation}`), { + headers + }) + if (!media.ok) throw new Error(`object read returned ${media.status}`) + return { generation: String(generation), object: await media.json().catch(() => null) } +} + +async function writeObject(fetchImpl, request, accessToken, audit) { + const boundary = `relay-cell-flags-${Date.now()}` + const metadata = { + name: `cells/${request.cellId}.json`, + contentType: 'application/json', + cacheControl: 'no-store', + // Who and which run, beside the generation the bucket's versioning already keeps. + metadata: audit + } + const body = [ + `--${boundary}`, + 'Content-Type: application/json; charset=UTF-8', + '', + JSON.stringify(metadata), + `--${boundary}`, + 'Content-Type: application/json', + '', + JSON.stringify(request.object), + `--${boundary}--`, + '' + ].join('\r\n') + const bucket = `${request.projectId}-relay-control` + const response = await fetchImpl( + `${STORAGE_ROOT}/upload/storage/v1/b/${bucket}/o?uploadType=multipart&ifGenerationMatch=${request.expectedGeneration}`, + { + method: 'POST', + headers: { + authorization: `Bearer ${accessToken}`, + 'content-type': `multipart/related; boundary=${boundary}` + }, + body + } + ) + if (response.status === 412) { + await response.body?.cancel().catch(() => undefined) + throw new Error('the object changed since expected-generation; re-read and retry') + } + if (!response.ok) throw new Error(`object write returned ${response.status}`) + return String((await response.json()).generation) +} + +const sameFlags = (left, right) => + left?.readinessLocal === right.readinessLocal && left?.ticketCheck === right.ticketCheck + +export async function operateCellFlags(request, dependencies) { + const { fetchImpl = fetch, accessToken, idToken, audit, log = console.log } = dependencies + const now = dependencies.now ?? Date.now + const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) + const runtime = await readRuntime(fetchImpl, request, idToken, sleep) + // Proves the origin is this cell, and that its image reads switch files at all. + if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) { + throw new Error(`origin answers as ${runtime.role}/${runtime.cellId}, not ${request.cellId}`) + } + if (!runtime.flagsApplied) throw new Error('cell image has no flag channel (no flagsApplied)') + const current = await readCurrentObject(fetchImpl, request, accessToken) + const plan = { + event: 'orca_relay_cell_flags_plan', + mode: request.mode, + cellId: request.cellId, + currentGeneration: current.generation, + currentObject: current.object, + appliedBefore: runtime.flagsApplied, + desired: request.object + } + // Before the generation check, so a dry run always shows the generation to pass next. + log(JSON.stringify(plan)) + if (current.generation !== request.expectedGeneration) { + throw new Error( + `current generation is ${current.generation}, not expected ${request.expectedGeneration}` + ) + } + if (request.mode === 'dry-run') return { ...plan, written: false } + const generation = await writeObject(fetchImpl, request, accessToken, audit) + log(JSON.stringify({ event: 'orca_relay_cell_flags_written', cellId: request.cellId, generation })) + const deadline = now() + READ_BACK_TIMEOUT_MS + let last = runtime.flagsApplied + while (now() < deadline) { + await sleep(READ_BACK_INTERVAL_MS) + try { + last = (await readRuntime(fetchImpl, request, idToken, sleep)).flagsApplied + } catch { + // The object is written; a failed poll only delays the read-back until the deadline. + continue + } + if (String(last?.generation) === generation && sameFlags(last.flags, request.object.flags)) { + const result = { ...plan, written: true, generation, applied: last } + log(JSON.stringify({ ...result, event: 'orca_relay_cell_flags_applied_read_back' })) + return result + } + } + throw new Error( + `cell did not apply generation ${generation} within ${READ_BACK_TIMEOUT_MS} ms ` + + `(last applied ${JSON.stringify(last)}); the object is written` + ) +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + const { values } = parseArgs({ + options: Object.fromEntries( + [ + 'mode', + 'cell-id', + 'cell-origin', + 'project-id', + 'readiness-local', + 'ticket-check', + 'expected-generation', + 'confirmation' + ].map((name) => [name, { type: 'string' }]) + ) + }) + try { + const request = parseCellFlagsRequest(values) + const accessToken = process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN + const idToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN + if (!accessToken || !idToken) throw new Error('both tokens are required') + await operateCellFlags(request, { + accessToken, + idToken, + audit: { + // The person who started this attempt, which differs from GITHUB_ACTOR on a re-run. + writtenBy: process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown', + runId: process.env.GITHUB_RUN_ID ?? 'local', + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0' + } + }) + } catch (error) { + // On stdout, so the job summary's copy of the run carries the reason. + console.log( + JSON.stringify({ + event: 'orca_relay_cell_flags_failed', + message: error instanceof Error ? error.message : String(error) + }) + ) + process.exitCode = 1 + } +} diff --git a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs new file mode 100644 index 00000000000..23834755e33 --- /dev/null +++ b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs @@ -0,0 +1,184 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + operateCellFlags, + parseCellFlagsRequest, + READ_BACK_TIMEOUT_MS, + WRITE_CONFIRMATION +} from './operate-relay-cell-flags.mjs' + +const CELL = 'production-gce-c26' +const values = (overrides = {}) => ({ + mode: 'write', + 'cell-id': CELL, + 'cell-origin': 'https://c26.relay.onorca.dev', + 'project-id': 'onorca-cloud', + 'readiness-local': 'true', + 'ticket-check': 'off', + 'expected-generation': '0', + confirmation: `${WRITE_CONFIRMATION} ${CELL}`, + ...overrides +}) + +// A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls. +function fakeGoogleAndCell({ stored = null, applyAfterPolls = 2, role = 'cell', cellId = CELL } = {}) { + const state = { stored, writes: [], polls: 0, applied: { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } } } + const fetchImpl = async (url, init = {}) => { + const target = new URL(url) + if (target.pathname === '/v1/admin/runtime-status') { + state.polls += 1 + if (state.stored && state.polls > applyAfterPolls) { + state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags } + } + return Response.json({ v: 1, role, cellId, flagsApplied: state.applied }) + } + if (target.pathname.startsWith('/upload/')) { + const expected = target.searchParams.get('ifGenerationMatch') + if (expected !== (state.stored?.generation ?? '0')) return new Response(null, { status: 412 }) + const parts = String(init.body).split('\r\n') + const object = JSON.parse(parts[parts.indexOf('Content-Type: application/json') + 2]) + const metadata = JSON.parse(parts[3]) + const generation = String(Number(state.stored?.generation ?? 1000) + 1) + state.stored = { generation, object } + state.writes.push({ generation, object, metadata }) + return Response.json({ generation }) + } + if (!state.stored) return new Response(null, { status: 404 }) + if (target.searchParams.get('alt') === 'media') return Response.json(state.stored.object) + return Response.json({ generation: state.stored.generation }) + } + return { state, fetchImpl } +} + +const run = (request, fake, extra = {}) => { + let clock = 0 + const lines = [] + return { + lines, + result: operateCellFlags(request, { + fetchImpl: fake.fetchImpl, + accessToken: 'access', + idToken: 'id', + audit: { writtenBy: 'octocat', runId: '42', runAttempt: '1' }, + log: (line) => lines.push(JSON.parse(line)), + now: () => clock, + sleep: async (ms) => { + clock += ms + }, + ...extra + }) + } +} + +test('refuses malformed requests before touching anything', () => { + assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/) + assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/) + assert.throws(() => parseCellFlagsRequest(values({ 'ticket-check': 'enforce' })), /ticket-check/) + assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/) + assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/) + assert.throws( + () => parseCellFlagsRequest(values({ 'cell-origin': 'https://c27.relay.onorca.dev' })), + /another cell/ + ) + assert.throws(() => parseCellFlagsRequest(values({ 'cell-origin': 'https://evil.example' })), /origin/) + // A dry run needs no confirmation. + assert.equal(parseCellFlagsRequest(values({ mode: 'dry-run', confirmation: '' })).mode, 'dry-run') +}) + +test('a dry run reports the change and writes nothing', async () => { + const fake = fakeGoogleAndCell() + const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake) + const outcome = await result + assert.equal(outcome.written, false) + assert.equal(fake.state.writes.length, 0) + assert.equal(lines[0].event, 'orca_relay_cell_flags_plan') + assert.deepEqual(lines[0].desired.flags, { readinessLocal: true, ticketCheck: 'off' }) +}) + +test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => { + const fake = fakeGoogleAndCell() + const { result, lines } = run(parseCellFlagsRequest(values()), fake) + const outcome = await result + assert.equal(outcome.written, true) + assert.equal(outcome.generation, '1001') + assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true, ticketCheck: 'off' } }) + assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' }) + assert.deepEqual(fake.state.writes[0].object, { + v: 1, + cellId: CELL, + flags: { readinessLocal: true, ticketCheck: 'off' } + }) + assert.deepEqual(lines.map((line) => line.event), [ + 'orca_relay_cell_flags_plan', + 'orca_relay_cell_flags_written', + 'orca_relay_cell_flags_applied_read_back' + ]) + // Flip back: the next write must name the generation just written. + const back = run( + parseCellFlagsRequest(values({ 'readiness-local': 'false', 'expected-generation': '1001' })), + fake + ) + assert.equal((await back.result).generation, '1002') +}) + +test('refuses a stale expected generation, and a concurrent writer between read and write', async () => { + const fake = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } } + }) + await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /current generation is 7/) + assert.equal(fake.state.writes.length, 0) + const racing = fakeGoogleAndCell() + const write = racing.fetchImpl + racing.fetchImpl = async (url, init) => { + if (new URL(url).pathname.startsWith('/upload/')) racing.state.stored = { generation: '9', object: {} } + return await write(url, init) + } + await assert.rejects(run(parseCellFlagsRequest(values()), racing).result, /changed since/) +}) + +test('a dry run against a stale generation still shows the plan, then refuses', async () => { + const fake = fakeGoogleAndCell({ + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } } + }) + const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake) + await assert.rejects(result, /current generation is 7/) + assert.equal(lines[0].event, 'orca_relay_cell_flags_plan') + assert.equal(lines[0].currentGeneration, '7') +}) + +test('rides out a transient 5xx on the first read and during the read-back', async () => { + const fake = fakeGoogleAndCell() + const inner = fake.fetchImpl + let runtimeCalls = 0 + fake.fetchImpl = async (url, init) => { + if (new URL(url).pathname === '/v1/admin/runtime-status') { + runtimeCalls += 1 + // The first call, and the first read-back poll after the write. + if (runtimeCalls === 1 || runtimeCalls === 3) return new Response(null, { status: 503 }) + } + return await inner(url, init) + } + const outcome = await run(parseCellFlagsRequest(values()), fake).result + assert.equal(outcome.written, true) + assert.equal(outcome.generation, '1001') +}) + +test('refuses an origin that is not the named cell, or an image without the flag channel', async () => { + await assert.rejects( + run(parseCellFlagsRequest(values()), fakeGoogleAndCell({ cellId: 'production-gce-c25' })).result, + /not production-gce-c26/ + ) + const old = fakeGoogleAndCell() + const fetchImpl = old.fetchImpl + old.fetchImpl = async (url, init) => + new URL(url).pathname === '/v1/admin/runtime-status' + ? Response.json({ v: 1, role: 'cell', cellId: CELL }) + : await fetchImpl(url, init) + await assert.rejects(run(parseCellFlagsRequest(values()), old).result, /no flag channel/) +}) + +test('fails loudly when the cell does not apply the write within the read-back window', async () => { + const fake = fakeGoogleAndCell({ applyAfterPolls: READ_BACK_TIMEOUT_MS }) + await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/) + assert.equal(fake.state.writes.length, 1) +}) diff --git a/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs b/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs index 3845cb90061..5e7671523b7 100644 --- a/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs +++ b/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs @@ -31,6 +31,7 @@ test('no workflow names the retired generic production deploy identity', async ( 'deploy-relay-production-same-cap.yml', 'deploy-relay-production.yml', 'operate-relay-asia-admission.yml', + 'operate-relay-production-cell-flags.yml', 'operate-relay-production-rehome-job.yml', 'publish-relay-production.yml' ].map((name) => relayWorkflowFile(name)).sort()) diff --git a/cloud/dev/scripts/relay-public-workflow-contract.test.mjs b/cloud/dev/scripts/relay-public-workflow-contract.test.mjs index d25ffb221f4..486914aaf3e 100644 --- a/cloud/dev/scripts/relay-public-workflow-contract.test.mjs +++ b/cloud/dev/scripts/relay-public-workflow-contract.test.mjs @@ -20,7 +20,7 @@ const UNGATED = relayWorkflowFile('verify.yml') const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED) test('the copy carries every relay workflow', () => { - assert.equal(relayWorkflows().length, 25) + assert.equal(relayWorkflows().length, 26) }) // Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming diff --git a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs index 64791605549..5bdc40fcd78 100644 --- a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs +++ b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs @@ -34,7 +34,7 @@ const EXPECTED_CONDITIONS = { "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'", github: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))", github_monitor: "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'", github_fence: diff --git a/cloud/infra/terraform/relay-github-actions.tf b/cloud/infra/terraform/relay-github-actions.tf index 186d8942f3d..fe4108312e4 100644 --- a/cloud/infra/terraform/relay-github-actions.tf +++ b/cloud/infra/terraform/relay-github-actions.tf @@ -19,6 +19,7 @@ locals { "deploy-relay-production-multi-target.yml", "deploy-relay-production.yml", "operate-relay-asia-admission.yml", + "operate-relay-production-cell-flags.yml", "publish-relay-production.yml" ] diff --git a/cloud/package.json b/cloud/package.json index e3a585ff858..63b20f20b74 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -22,7 +22,7 @@ "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", - "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck" }, "devDependencies": {