mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
fix(relay): check clock skew on every cell production.tfvars declares (#28786)
* Monitor clock skew on every relay cell in production.tfvars * test(cloud): run the clock-skew monitor workflow test in Cloud Verify --------- Co-authored-by: Neil <neil@stably.ai>
This commit is contained in:
Žan Mlakar
Neil
@@ -23,8 +23,13 @@ jobs:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: cloud/infra/terraform/environments/production.tfvars
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
|
||||
- name: Measure Date-header skew for every relay cell
|
||||
working-directory: .
|
||||
run: |
|
||||
set -u
|
||||
# Date headers carry whole seconds; compare floored seconds on both
|
||||
@@ -36,10 +41,18 @@ jobs:
|
||||
failures=0
|
||||
reachable=0
|
||||
unserved=0
|
||||
# Keep this upper bound at or above the highest provisioned cell in
|
||||
# environments/production.tfvars; unlisted cells are silently unmonitored.
|
||||
for n in $(seq 1 22); do
|
||||
cell="c${n}"
|
||||
# Every relay_gce_cells hostname (the block relayOpsCellsFromTerraform reads), fenced
|
||||
# cells included, so a newly declared cell is probed from its first run.
|
||||
cells="$(awk '
|
||||
/^relay_gce_cells[ \t]*=[ \t]*\{/ { inside = 1; next }
|
||||
inside && /^\}/ { exit }
|
||||
inside && /^[ \t]*hostname[ \t]*=/ { split($0, quoted, "\""); print quoted[2] }
|
||||
' infra/terraform/environments/production.tfvars || true)"
|
||||
if [ -z "${cells}" ]; then
|
||||
echo "::error::read no relay cells from production.tfvars; monitor blind"
|
||||
exit 1
|
||||
fi
|
||||
for cell in ${cells}; do
|
||||
url="https://${cell}.relay.onorca.dev/health"
|
||||
# Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own
|
||||
# accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { delimiter, join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import { readRelayWorkflow, relayTreePath } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readRelayWorkflow('monitor-relay-clock-skew.yml')
|
||||
const productionTfvars = readFileSync(
|
||||
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
// The measuring step's `run: |` block, unindented as the runner receives it.
|
||||
function measureScript() {
|
||||
const lines = workflow.split('\n')
|
||||
const step = lines.findIndex((line) => line.includes('name: Measure Date-header skew'))
|
||||
const start = lines.findIndex((line, index) => index > step && line.trim() === 'run: |') + 1
|
||||
assert.ok(step !== -1 && start > 0, 'measuring step not found')
|
||||
const end = lines.findIndex(
|
||||
(line, index) => index >= start && line.trim() !== '' && !line.startsWith(' '.repeat(10))
|
||||
)
|
||||
return lines.slice(start, end === -1 ? undefined : end).map((line) => line.slice(10)).join('\n')
|
||||
}
|
||||
|
||||
// Runs the step in a checkout holding `tfvars`, with every cell answering like a fenced one so
|
||||
// each probe is named without a clock to judge.
|
||||
function runMonitor(tfvars) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'relay-clock-skew-'))
|
||||
try {
|
||||
const bin = join(root, 'bin')
|
||||
mkdirSync(bin)
|
||||
writeFileSync(join(bin, 'curl'), '#!/usr/bin/env bash\nprintf "HTTP/2 404 \\r\\n\\r\\n"\n')
|
||||
chmodSync(join(bin, 'curl'), 0o755)
|
||||
const environments = join(root, 'cloud', 'infra', 'terraform', 'environments')
|
||||
mkdirSync(environments, { recursive: true })
|
||||
if (tfvars !== undefined) writeFileSync(join(environments, 'production.tfvars'), tfvars)
|
||||
return spawnSync('bash', ['-e', '-c', measureScript()], {
|
||||
cwd: join(root, 'cloud'),
|
||||
env: { ...process.env, PATH: `${bin}${delimiter}${process.env.PATH}` },
|
||||
encoding: 'utf8'
|
||||
})
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
test('checks out the tfvars the measuring step reads', () => {
|
||||
const tfvars = relayTreePath('infra/terraform/environments/production.tfvars')
|
||||
assert.ok(workflow.includes(`sparse-checkout: ${tfvars}\n`))
|
||||
assert.ok(workflow.includes('sparse-checkout-cone-mode: false\n'))
|
||||
})
|
||||
|
||||
test('probes every cell production.tfvars declares', () => {
|
||||
const declared = [
|
||||
...productionTfvars.matchAll(/"production-gce-c\d+"\s*=\s*\{([\s\S]*?)\n {2}\}/g)
|
||||
].map(([, body]) => /\bhostname\s*=\s*"([^"]+)"/.exec(body)?.[1])
|
||||
assert.ok(declared.length > 0)
|
||||
const result = runMonitor(productionTfvars)
|
||||
assert.deepEqual(
|
||||
[...result.stdout.matchAll(/^(\S+): not serving/gm)].map(([, cell]) => cell),
|
||||
declared
|
||||
)
|
||||
})
|
||||
|
||||
test('fails instead of probing nothing when no cell can be read', () => {
|
||||
for (const tfvars of [undefined, 'relay_gce_cells = {}\n']) {
|
||||
const result = runMonitor(tfvars)
|
||||
assert.equal(result.status, 1)
|
||||
assert.match(result.stdout, /::error::read no relay cells/)
|
||||
}
|
||||
})
|
||||
+1
-1
@@ -17,7 +17,7 @@
|
||||
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
|
||||
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs && pnpm test:runner",
|
||||
"test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-clock-skew-monitor-workflow.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"typecheck": "pnpm -r typecheck",
|
||||
"test:runner": "node --import tsx --conditions=orca-source --test dev/scripts/run-relay-tests.test.mjs"
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user