diff --git a/.github/scripts/build-ci-matrix.js b/.github/scripts/build-ci-matrix.js index f69398d76a..394c0c68a7 100644 --- a/.github/scripts/build-ci-matrix.js +++ b/.github/scripts/build-ci-matrix.js @@ -115,6 +115,13 @@ for (const s of include) { // silently DROPS older rows while still returning a success-shaped response. A // suite that seeds history — SLOs measure a rolling 7-day window — needs a wider // one, and scoping it to those shards keeps every other shard on the default. +// +// slo_backfill_chunk_secs follows the same rule for the same class of reason: +// it's how much history one SLO backfill chunk covers, read once at server +// start. The workflow default (86400 = 1 day) makes a 7-day-window test SLO +// backfill in 7 sequential chunks under ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1; +// SLO-Measurement widens it to the full window so its 3 backfilled SLOs don't +// pay 21 chunks serially. const matrix = include.map((s) => ({ testfolder: s.testfolder, actual_folder: s.actual_folder, @@ -126,6 +133,10 @@ const matrix = include.map((s) => ({ s.ingest_allowed_upto === undefined || s.ingest_allowed_upto === null ? "" : String(s.ingest_allowed_upto), + slo_backfill_chunk_secs: + s.slo_backfill_chunk_secs === undefined || s.slo_backfill_chunk_secs === null + ? "" + : String(s.slo_backfill_chunk_secs), // Per-shard worker count. Empty = use playwright.config.js (5 in CI). // // `fullyParallel` runs separate spec FILES concurrently, and diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 5cd4d1ebd5..0d8441bc07 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -32,6 +32,7 @@ env: ZO_QUICK_MODE_STRATEGY: first ZO_ALLOW_USER_DEFINED_SCHEMAS: true ZO_INGEST_ALLOWED_UPTO: 5 + ZO_SLO_BACKFILL_CHUNK_SECS: 86400 # matches the server default; per-shard override below ZO_FEATURE_QUERY_EXCLUDE_ALL: false ZO_USAGE_BATCH_SIZE: 200 ZO_USAGE_PUBLISH_INTERVAL: 2 @@ -259,6 +260,15 @@ jobs: # Set via "ingest_allowed_upto" in ci-matrix/ci_matrix.json; shards that # omit it keep the workflow default. ZO_INGEST_ALLOWED_UPTO: ${{ matrix.ingest_allowed_upto || env.ZO_INGEST_ALLOWED_UPTO }} + # Same per-shard rule: how much history one SLO backfill chunk covers. + # The production default (86400 = 1 day) makes a 7-day-window test SLO + # backfill in 7 sequential chunks under ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1 + # (SLO-Measurement's shard note above) — no test here asserts on + # per-chunk timing or partial coverage, only on the final measured + # state, so a shard-only larger chunk is free wall-clock, not a + # behavior change. Set via "slo_backfill_chunk_secs" in + # ci-matrix/ci_matrix.json; shards that omit it keep the server default. + ZO_SLO_BACKFILL_CHUNK_SECS: ${{ matrix.slo_backfill_chunk_secs || env.ZO_SLO_BACKFILL_CHUNK_SECS }} run: chmod +x ./release-ci-binary/openobserve && ./release-ci-binary/openobserve > o2.log 2>&1 & - name: Wait for start diff --git a/tests/ui-testing/ci-matrix/README.md b/tests/ui-testing/ci-matrix/README.md index 4c24e73534..4d0a39f9c6 100644 --- a/tests/ui-testing/ci-matrix/README.md +++ b/tests/ui-testing/ci-matrix/README.md @@ -64,6 +64,13 @@ The last three exist because a shard is the smallest unit that can change them. `workers: 1`; the specs cannot express that themselves. `SLO-Measurement` does, because both of its specs wait on the SLO backfill job, which runs with `ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1`. +- `slo_backfill_chunk_secs` — the server default is **86400** (1 day), so a test + SLO with a 7-day window backfills in 7 sequential chunks under the concurrency-1 + rule above. `SLO-Measurement` sets `604800` so each of its 7-day-window SLOs + backfills in a single chunk instead. Safe only because no spec in that shard + asserts on per-chunk timing or partial coverage — only on the final measured + state. If a future spec in this shard needs to see backfill mid-flight, give + that spec its own shard rather than lowering this back down. ## Disabling a spec (JSON has no `//` comments) diff --git a/tests/ui-testing/ci-matrix/ci_matrix.json b/tests/ui-testing/ci-matrix/ci_matrix.json index ab9e3e2f45..ddddc8a734 100644 --- a/tests/ui-testing/ci-matrix/ci_matrix.json +++ b/tests/ui-testing/ci-matrix/ci_matrix.json @@ -436,10 +436,11 @@ }, { "testfolder": "SLO-Measurement", - "_comment": "SEPARATE SHARD ON PURPOSE. These two specs seed 8 days of history and then wait for the backfill job, which fills roughly one day-chunk per 40s with ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1. Any spec creating SLOs on the same instance queues behind them and starves the wait, so they must not share a runner with the SLO shard. Both specs are mode:serial and pay the backfill once in beforeAll (~5 min each).", + "_comment": "SEPARATE SHARD ON PURPOSE. These two specs seed 8 days of history and wait for the backfill job (ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1, so its 3 backfilled SLOs queue strictly one at a time). slo_backfill_chunk_secs is bumped from the 86400 (1-day) server default to the full 604800 (7-day) window each of those SLOs uses, so each finishes in 1 chunk instead of 7 -- no test here asserts on per-chunk timing or partial coverage, only on the final measured state. Any spec creating SLOs on the same instance queues behind them and starves the wait, so they must not share a runner with the SLO shard. Both specs are mode:serial and pay the backfill once in beforeAll.", "actual_folder": "SLO", "browser": "chrome", "ingest_allowed_upto": "240", + "slo_backfill_chunk_secs": "604800", "workers": 1, "run_files": [ "slo-timeslice.spec.js",