From 457a3f317e05bd21518798186cc796efb01b1179 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 27 Jul 2026 19:05:25 +0800 Subject: [PATCH] ci: add heavy query regression label (#8619) Signed-off-by: discord9 --- .../query-regression/README.md | 33 ++++++----- .github/scripts/query-regression-run.py | 27 +++++---- .github/workflows/query-regression.yml | 15 ++--- tests/perf/README.md | 18 +++--- .../test_query_regression_case_selection.py | 57 +++++++++++++++++++ 5 files changed, 111 insertions(+), 39 deletions(-) create mode 100644 tests/perf/test_query_regression_case_selection.py diff --git a/.github/runner-scale-sets/query-regression/README.md b/.github/runner-scale-sets/query-regression/README.md index 307d42396f..d145771989 100644 --- a/.github/runner-scale-sets/query-regression/README.md +++ b/.github/runner-scale-sets/query-regression/README.md @@ -30,21 +30,24 @@ kubectl -n arc-runners create secret generic greptimedb-arc-github-app \ The values files here reference that secret by name. -A maintainer applying the `query-regression` label is **trust admission for -that exact PR revision**. The admitted job may use this scale set's dedicated, -writable persistent cache. `pull_request: labeled` is the only PR trigger: the -label event snapshots its merge, head, and base SHAs. A queued job fetches that -immutable event merge SHA directly, verifies it is a two-parent merge whose -parents include the snapshotted head exactly once, and uses its other parent as -the actual base build revision. The snapshotted event base is retained for audit -only, so a difference from the merge's non-head parent is not a failure. The job -never follows a newer mutable PR merge ref. An unavailable event merge, or one -that does not contain exactly one snapshotted head parent, fails closed. A later -PR head change does not retarget an already queued run: it may execute only its -previously trusted event revision if that revision remains fetchable. To run the -new revision, the maintainer must review it, remove the label, and re-add -`query-regression`; cancel the old run if it is no longer wanted. An existing -label does not automatically rerun the benchmark. +A maintainer applying the `query-regression` or `heavy-regression` label is +**trust admission for that exact PR revision**. `query-regression` runs the five +routine default cases; `heavy-regression` runs only the high-cardinality +`prom_remote_write_7913` remote-write case. The admitted job may use this scale +set's dedicated, writable persistent cache. `pull_request: labeled` is the only +PR trigger: the label event snapshots its merge, head, and base SHAs. A queued +job fetches that immutable event merge SHA directly, verifies it is a two-parent +merge whose parents include the snapshotted head exactly once, and uses its +other parent as the actual base build revision. The snapshotted event base is +retained for audit only, so a difference from the merge's non-head parent is +not a failure. The job never follows a newer mutable PR merge ref. An +unavailable event merge, or one that does not contain exactly one snapshotted +head parent, fails closed. A later PR head change does not retarget an already +queued run: it may execute only its previously trusted event revision if that +revision remains fetchable. To run the new revision, the maintainer must review +it, remove the label, and re-add the desired regression label; cancel the old +run if it is no longer wanted. An existing label does not automatically rerun +the benchmark. Admission does not relax runner hardening or GitHub permissions. Keep service-account token mounting disabled; do not mount host paths, the Docker diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index aa30aab4fb..8bf78ca2f0 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -30,23 +30,30 @@ DEFAULT_CASES = [ "tests/perf/query_cases/prom_remote_write_run_heavy/case.toml", "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", "tests/perf/query_cases/prom_remote_write_integer_counter/case.toml", - "tests/perf/query_cases/promql_pushdown_7913/case.toml", - "tests/perf/query_cases/analyze_verbose_many_files/case.toml", - "tests/perf/query_cases/sql_topk_order_by/case.toml", - "tests/perf/query_cases/sql_aggregate_order_by/case.toml", - "tests/perf/query_cases/sql_join_filter_order/case.toml", ] +HEAVY_CASES = [ + "tests/perf/query_cases/prom_remote_write_7913/case.toml", +] + +CASE_GROUPS = { + "all": DEFAULT_CASES, + "heavy": HEAVY_CASES, +} + def split_cases(values: list[str]) -> list[str]: tokens: list[str] = [] for value in values: tokens.extend(part for part in re.split(r"[\s,]+", value.strip()) if part) - if not tokens or tokens == ["all"]: + if not tokens: return DEFAULT_CASES.copy() - if "all" in tokens: - raise ValueError("'all' cannot be mixed with explicit case paths") - return list(dict.fromkeys(tokens)) + if "all" in tokens and len(tokens) > 1: + raise ValueError("'all' cannot be mixed with other case selectors") + cases: list[str] = [] + for token in tokens: + cases.extend(CASE_GROUPS.get(token, [token])) + return list(dict.fromkeys(cases)) def parse_bool(value: str) -> bool: @@ -151,7 +158,7 @@ def write_summary(args: argparse.Namespace, reports: list[Path]) -> int: def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--cases", action="append", help="'all' or comma/space separated case paths") + parser.add_argument("--cases", action="append", help="'all', 'heavy', or comma/space separated case paths") parser.add_argument("--base-src", type=Path, default=Path("base-src")) parser.add_argument("--candidate-src", type=Path, default=Path("candidate-src")) parser.add_argument("--base-bin", type=Path, default=configured_path(os.environ.get("BASE_BIN"))) diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index 63387bc310..fd15e65960 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: case: - description: Query perf case path(s) in candidate checkout, or all + description: Query perf case path(s) in candidate checkout, all, or heavy required: false type: string default: all @@ -39,7 +39,7 @@ on: workflow_dispatch: inputs: case: - description: Query perf case path(s) in candidate checkout, or all + description: Query perf case path(s) in candidate checkout, all, or heavy required: true default: all base_ref: @@ -86,8 +86,8 @@ jobs: ${{ github.event_name != 'pull_request' || (github.event_name == 'pull_request' && !github.event.pull_request.draft && - contains(github.event.pull_request.labels.*.name, 'query-regression') && - (github.event.action != 'labeled' || github.event.label.name == 'query-regression')) }} + (github.event.label.name == 'query-regression' || + github.event.label.name == 'heavy-regression')) }} runs-on: ${{ github.event_name != 'pull_request' && inputs.runner || 'perf-regression-8-cores' }} timeout-minutes: 180 concurrency: @@ -134,7 +134,7 @@ jobs: fail_closed() { printf '%s\n' \ - "PR changed or merge result unavailable; maintainer must remove and re-add query-regression after reviewing current revision. $1" >&2 + "PR changed or merge result unavailable; maintainer must remove and re-add the regression label after reviewing current revision. $1" >&2 exit 1 } @@ -573,6 +573,7 @@ jobs: uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py uv run --no-project python tests/perf/test_query_regression_runner_otlp_trace_load.py uv run --no-project python tests/perf/test_query_regression_summary_otlp.py + uv run --no-project python tests/perf/test_query_regression_case_selection.py - name: Build candidate greptime and fixture generators working-directory: src @@ -598,11 +599,11 @@ jobs: - name: Run query regression id: run env: - CASE_PATHS: ${{ github.event_name == 'pull_request' && 'all' || inputs.case }} + CASE_PATHS: ${{ github.event_name == 'pull_request' && (github.event.label.name == 'heavy-regression' && 'heavy' || 'all') || inputs.case }} HTTP_TIMEOUT: ${{ github.event_name == 'pull_request' && '300' || inputs.http_timeout }} ALLOW_LARGE_FIXTURE: ${{ github.event_name == 'pull_request' && 'true' || inputs.allow_large_fixture }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - CASE_NAME: ${{ github.event_name == 'pull_request' && 'default case set' || inputs.case }} + CASE_NAME: ${{ github.event_name == 'pull_request' && (github.event.label.name == 'heavy-regression' && 'heavy case set' || 'default case set') || inputs.case }} BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture diff --git a/tests/perf/README.md b/tests/perf/README.md index ce994b6f52..c335f0a43b 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -205,8 +205,10 @@ experiment matrix for those policy comparisons. `tests/perf/query_cases/prom_remote_write_7913/case.toml` is a larger manual case for issue #7913. It writes 8192 series × 20160 samples through remote-write in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d -TQL selectors. It is not included in the default case set because ingestion cost -dominates routine CI validation. +TQL selectors. It is not included in the default `all` case set because ingestion +cost dominates routine CI validation. Adding the `heavy-regression` PR label runs +only this case; `query-regression` runs the five routine default cases. Manual +workflow dispatch accepts the `heavy` token to select this case. ## OTLP trace load scenario @@ -469,11 +471,13 @@ binaries. Candidate `query_perf_fixture` is the extra head-side helper binary; the runner uses candidate `greptime datanode parquetbench/scanbench` as the read-bench tool against each target's data directory. -The workflow runs automatically only when `query-regression` is added to a -non-draft PR; it does not rerun on pushes, ready-for-review, or reopen events. -PR runs build base/candidate once and then run the default case set with -`--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`, one case -path, or a comma/whitespace-separated list of case paths, and can override refs. +The workflow runs automatically only when `query-regression` or `heavy-regression` +is added to a non-draft PR; it does not rerun on pushes, ready-for-review, or +reopen events. `query-regression` runs the five routine default cases, while +`heavy-regression` runs only the high-cardinality remote-write #7913 case. PR runs +build base/candidate once and use `--allow-large-fixture`. Manual +`workflow_dispatch` runs can pass `all`, `heavy`, one case path, or a +comma/whitespace-separated list of case paths, and can override refs. The main report artifact uploads only aggregate/per-target JSON reports, component logs, and `query-regression-summary.md` with seven-day retention; fixture data, SSTs, and cluster state are excluded. PR runs also upload a diff --git a/tests/perf/test_query_regression_case_selection.py b/tests/perf/test_query_regression_case_selection.py new file mode 100644 index 0000000000..a240216fd3 --- /dev/null +++ b/tests/perf/test_query_regression_case_selection.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Coverage for query-regression case group selection.""" + +import importlib.util +import sys +import unittest +from pathlib import Path + + +RUNNER_PATH = Path(__file__).parents[2] / ".github/scripts/query-regression-run.py" +SPEC = importlib.util.spec_from_file_location("query_regression_run_under_test", RUNNER_PATH) +assert SPEC is not None and SPEC.loader is not None +runner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = runner +SPEC.loader.exec_module(runner) + + +class QueryRegressionCaseSelectionTest(unittest.TestCase): + def test_all_selects_the_five_routine_cases(self) -> None: + self.assertEqual( + runner.split_cases(["all"]), + [ + "tests/perf/query_cases/smoke_direct_sst/case.toml", + "tests/perf/query_cases/prom_remote_write_seeded_random/case.toml", + "tests/perf/query_cases/prom_remote_write_run_heavy/case.toml", + "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", + "tests/perf/query_cases/prom_remote_write_integer_counter/case.toml", + ], + ) + + def test_heavy_selects_only_remote_write_7913(self) -> None: + self.assertEqual( + runner.split_cases(["heavy"]), + ["tests/perf/query_cases/prom_remote_write_7913/case.toml"], + ) + + def test_explicit_paths_remain_selectable(self) -> None: + case = "tests/perf/query_cases/sql_topk_order_by/case.toml" + self.assertEqual(runner.split_cases([case]), [case]) + + +if __name__ == "__main__": + unittest.main()