diff --git a/.github/workflows/cloud-monitor-relay-clock-skew.yml b/.github/workflows/cloud-monitor-relay-clock-skew.yml index 9d4de846519..6bcca3f8124 100644 --- a/.github/workflows/cloud-monitor-relay-clock-skew.yml +++ b/.github/workflows/cloud-monitor-relay-clock-skew.yml @@ -23,8 +23,13 @@ jobs: if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: cloud/infra/terraform/environments/production.tfvars + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Measure Date-header skew for every relay cell - working-directory: . run: | set -u # Date headers carry whole seconds; compare floored seconds on both @@ -36,10 +41,18 @@ jobs: failures=0 reachable=0 unserved=0 - # Keep this upper bound at or above the highest provisioned cell in - # environments/production.tfvars; unlisted cells are silently unmonitored. - for n in $(seq 1 22); do - cell="c${n}" + # Every relay_gce_cells hostname (the block relayOpsCellsFromTerraform reads), fenced + # cells included, so a newly declared cell is probed from its first run. + cells="$(awk ' + /^relay_gce_cells[ \t]*=[ \t]*\{/ { inside = 1; next } + inside && /^\}/ { exit } + inside && /^[ \t]*hostname[ \t]*=/ { split($0, quoted, "\""); print quoted[2] } + ' infra/terraform/environments/production.tfvars || true)" + if [ -z "${cells}" ]; then + echo "::error::read no relay cells from production.tfvars; monitor blind" + exit 1 + fi + for cell in ${cells}; do url="https://${cell}.relay.onorca.dev/health" # Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own # accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as diff --git a/cloud/dev/scripts/relay-clock-skew-monitor-workflow.test.mjs b/cloud/dev/scripts/relay-clock-skew-monitor-workflow.test.mjs new file mode 100644 index 00000000000..cdd40a23c82 --- /dev/null +++ b/cloud/dev/scripts/relay-clock-skew-monitor-workflow.test.mjs @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { delimiter, join } from 'node:path' +import { test } from 'node:test' +import { readRelayWorkflow, relayTreePath } from './relay-repository.mjs' + +const workflow = readRelayWorkflow('monitor-relay-clock-skew.yml') +const productionTfvars = readFileSync( + new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), + 'utf8' +) + +// The measuring step's `run: |` block, unindented as the runner receives it. +function measureScript() { + const lines = workflow.split('\n') + const step = lines.findIndex((line) => line.includes('name: Measure Date-header skew')) + const start = lines.findIndex((line, index) => index > step && line.trim() === 'run: |') + 1 + assert.ok(step !== -1 && start > 0, 'measuring step not found') + const end = lines.findIndex( + (line, index) => index >= start && line.trim() !== '' && !line.startsWith(' '.repeat(10)) + ) + return lines.slice(start, end === -1 ? undefined : end).map((line) => line.slice(10)).join('\n') +} + +// Runs the step in a checkout holding `tfvars`, with every cell answering like a fenced one so +// each probe is named without a clock to judge. +function runMonitor(tfvars) { + const root = mkdtempSync(join(tmpdir(), 'relay-clock-skew-')) + try { + const bin = join(root, 'bin') + mkdirSync(bin) + writeFileSync(join(bin, 'curl'), '#!/usr/bin/env bash\nprintf "HTTP/2 404 \\r\\n\\r\\n"\n') + chmodSync(join(bin, 'curl'), 0o755) + const environments = join(root, 'cloud', 'infra', 'terraform', 'environments') + mkdirSync(environments, { recursive: true }) + if (tfvars !== undefined) writeFileSync(join(environments, 'production.tfvars'), tfvars) + return spawnSync('bash', ['-e', '-c', measureScript()], { + cwd: join(root, 'cloud'), + env: { ...process.env, PATH: `${bin}${delimiter}${process.env.PATH}` }, + encoding: 'utf8' + }) + } finally { + rmSync(root, { recursive: true, force: true }) + } +} + +test('checks out the tfvars the measuring step reads', () => { + const tfvars = relayTreePath('infra/terraform/environments/production.tfvars') + assert.ok(workflow.includes(`sparse-checkout: ${tfvars}\n`)) + assert.ok(workflow.includes('sparse-checkout-cone-mode: false\n')) +}) + +test('probes every cell production.tfvars declares', () => { + const declared = [ + ...productionTfvars.matchAll(/"production-gce-c\d+"\s*=\s*\{([\s\S]*?)\n {2}\}/g) + ].map(([, body]) => /\bhostname\s*=\s*"([^"]+)"/.exec(body)?.[1]) + assert.ok(declared.length > 0) + const result = runMonitor(productionTfvars) + assert.deepEqual( + [...result.stdout.matchAll(/^(\S+): not serving/gm)].map(([, cell]) => cell), + declared + ) +}) + +test('fails instead of probing nothing when no cell can be read', () => { + for (const tfvars of [undefined, 'relay_gce_cells = {}\n']) { + const result = runMonitor(tfvars) + assert.equal(result.status, 1) + assert.match(result.stdout, /::error::read no relay cells/) + } +}) diff --git a/cloud/package.json b/cloud/package.json index 8f7a8046b22..6701f3859fd 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -17,7 +17,7 @@ "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs && pnpm test:runner", - "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-clock-skew-monitor-workflow.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck", "test:runner": "node --import tsx --conditions=orca-source --test dev/scripts/run-relay-tests.test.mjs" },