feat: add query regression perf harness (#8406)

* feat: add query regression perf harness Signed-off-by: discord9 <discord9@163.com> * feat: extend query regression cases Signed-off-by: discord9 <discord9@163.com> * ci: harden query regression workflows Signed-off-by: discord9 <discord9@163.com> * fix: address query regression review comments Signed-off-by: discord9 <discord9@163.com> * ci: limit query regression PR triggers Signed-off-by: discord9 <discord9@163.com> * ci: run full query regression case set Signed-off-by: discord9 <discord9@163.com> * refactor: model query regression scenarios Signed-off-by: discord9 <discord9@163.com> * fix: avoid unenforced query regression thresholds Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com>
2026-07-05 21:40:38 +00:00 · 2026-07-03 17:09:01 +08:00
parent 424e400c80
commit c44f8da646
18 changed files with 2967 additions and 0 deletions
--- a/.github/scripts/query-regression-pr-metadata.py
+++ b/.github/scripts/query-regression-pr-metadata.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Write metadata consumed by the trusted query-regression comment workflow."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", default=Path("query-regression-pr.json"), type=Path)
+    args = parser.parse_args()
+
+    metadata = {
+        "pr_number": int(os.environ["PR_NUMBER"]),
+        "head_sha": os.environ["HEAD_SHA"],
+        "head_repo": os.environ["HEAD_REPO"],
+        "base_repo": os.environ["BASE_REPO"],
+        "run_id": int(os.environ["RUN_ID"]),
+        "run_attempt": int(os.environ["RUN_ATTEMPT"]),
+    }
+    args.output.write_text(json.dumps(metadata, sort_keys=True) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/.github/scripts/query-regression-run.py
+++ b/.github/scripts/query-regression-run.py
@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run one or more query regression cases after the binaries are built."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+from pathlib import Path
+
+
+DEFAULT_CASES = [
+    "tests/perf/query_cases/smoke_direct_sst/case.toml",
+    "tests/perf/query_cases/promql_pushdown_7913/case.toml",
+    "tests/perf/query_cases/sql_topk_order_by/case.toml",
+    "tests/perf/query_cases/sql_aggregate_order_by/case.toml",
+    "tests/perf/query_cases/sql_join_filter_order/case.toml",
+]
+
+
+def split_cases(values: list[str]) -> list[str]:
+    tokens: list[str] = []
+    for value in values:
+        tokens.extend(part for part in re.split(r"[\s,]+", value.strip()) if part)
+    if not tokens or tokens == ["all"]:
+        return DEFAULT_CASES.copy()
+    if "all" in tokens:
+        raise ValueError("'all' cannot be mixed with explicit case paths")
+    return list(dict.fromkeys(tokens))
+
+
+def parse_bool(value: str) -> bool:
+    return value.lower() in {"1", "true", "yes", "on"}
+
+
+def profile_dir(cargo_profile: str) -> str:
+    if cargo_profile == "dev":
+        return "debug"
+    return cargo_profile
+
+
+def resolve_case_path(candidate_src: Path, case: str) -> Path:
+    path = Path(case)
+    if path.is_absolute() or path.parts[:1] == (candidate_src.name,):
+        return path
+    return candidate_src / path
+
+
+def case_slug(case_path: Path) -> str:
+    raw = case_path.parent.name if case_path.name == "case.toml" else case_path.stem
+    return re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip("-") or "case"
+
+
+def append_github_output(path: str | None, status: int) -> None:
+    if not path:
+        return
+    with open(path, "a", encoding="utf-8") as fp:
+        fp.write(f"status={status}\n")
+
+
+def append_step_summary(summary: Path) -> None:
+    step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
+    if not step_summary or not summary.exists():
+        return
+    with open(step_summary, "a", encoding="utf-8") as out:
+        out.write(summary.read_text())
+
+
+def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int:
+    target_dir = profile_dir(args.cargo_profile)
+    cmd = [
+        "uv",
+        "run",
+        "--no-project",
+        "python",
+        str(args.candidate_src / "tests/perf/query_regression_runner.py"),
+        "--case",
+        str(case_path),
+        "--base-bin",
+        str(args.base_src / "target" / target_dir / "greptime"),
+        "--candidate-bin",
+        str(args.candidate_src / "target" / target_dir / "greptime"),
+        "--fixture-generator",
+        str(args.candidate_src / "target" / target_dir / "query_perf_fixture"),
+        "--work-dir",
+        str(work_dir),
+        "--http-timeout",
+        str(args.http_timeout),
+    ]
+    if parse_bool(args.allow_large_fixture):
+        cmd.append("--allow-large-fixture")
+
+    print(f"::group::Query regression case: {case_path}", flush=True)
+    try:
+        return subprocess.run(cmd, check=False).returncode
+    finally:
+        print("::endgroup::", flush=True)
+
+
+def write_summary(args: argparse.Namespace, reports: list[Path]) -> int:
+    cmd = ["uv", "run", "--no-project", "python", str(args.summary_script)]
+    for report in reports:
+        cmd.extend(["--report", str(report)])
+    cmd.extend(
+        [
+            "--run-url",
+            args.run_url,
+            "--case-name",
+            args.case_name,
+            "--base-ref",
+            args.base_ref,
+            "--candidate-ref",
+            args.candidate_ref,
+            "--output",
+            str(args.summary_output),
+        ]
+    )
+    return subprocess.run(cmd, check=False).returncode
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cases", action="append", help="'all' or comma/space separated case paths")
+    parser.add_argument("--base-src", type=Path, default=Path("base-src"))
+    parser.add_argument("--candidate-src", type=Path, default=Path("candidate-src"))
+    parser.add_argument("--cargo-profile", default=os.environ.get("CARGO_PROFILE", "nightly"))
+    parser.add_argument("--work-dir", default=Path("query-regression-work"), type=Path)
+    parser.add_argument("--http-timeout", default=os.environ.get("HTTP_TIMEOUT", "300"))
+    parser.add_argument("--allow-large-fixture", default=os.environ.get("ALLOW_LARGE_FIXTURE", "false"))
+    parser.add_argument(
+        "--summary-script",
+        type=Path,
+        default=Path("candidate-src/.github/scripts/query-regression-summary.py"),
+    )
+    parser.add_argument("--summary-output", type=Path, default=Path("query-regression-summary.md"))
+    parser.add_argument("--run-url", default=os.environ.get("RUN_URL", ""))
+    parser.add_argument("--case-name", default=os.environ.get("CASE_NAME", "default case set"))
+    parser.add_argument("--base-ref", default=os.environ.get("BASE_REF", ""))
+    parser.add_argument("--candidate-ref", default=os.environ.get("CANDIDATE_REF", ""))
+    parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT"))
+    args = parser.parse_args()
+
+    try:
+        cases = split_cases(args.cases or [os.environ.get("CASE_PATHS", "all")])
+    except ValueError as err:
+        print(f"error: {err}", flush=True)
+        append_github_output(args.github_output, 1)
+        return 0
+
+    reports: list[Path] = []
+    status = 0
+    for case in cases:
+        case_path = resolve_case_path(args.candidate_src, case)
+        work_dir = args.work_dir / case_slug(case_path)
+        reports.append(work_dir / "query-regression-report.json")
+        case_status = run_case(args, case_path, work_dir)
+        if case_status != 0:
+            status = case_status or 1
+
+    summary_status = write_summary(args, reports)
+    if summary_status != 0 and status == 0:
+        status = summary_status
+    append_step_summary(args.summary_output)
+    append_github_output(args.github_output, status)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/.github/scripts/query-regression-summary.py
+++ b/.github/scripts/query-regression-summary.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Format tests/perf query regression JSON reports as GitHub Markdown."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+def fmt_ms(value: Any) -> str:
+    if value is None:
+        return "N/A"
+    try:
+        return f"{float(value):.2f}"
+    except (TypeError, ValueError):
+        return "N/A"
+
+
+def esc(value: Any) -> str:
+    text = "N/A" if value is None else str(value)
+    return text.replace("|", "\\|").replace("\n", " ")
+
+
+def status_emoji(status: str | None) -> str:
+    return {"ok": "✅", "measured": "✅", "failed": "❌", "planned": "📝", "fixture-ready": "🧪"}.get(
+        status or "", "⚠️"
+    )
+
+
+def measurement_map(target: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    return {m.get("name") or f"query-{i}": m for i, m in enumerate(target.get("measurements", []))}
+
+
+def regression_pct(base: Any, candidate: Any) -> str:
+    try:
+        b = float(base)
+        c = float(candidate)
+    except (TypeError, ValueError):
+        return "N/A"
+    if b == 0:
+        return "N/A"
+    return f"{(c - b) / b * 100:+.1f}%"
+
+
+def threshold_status(thresholds: list[dict[str, Any]], query: str) -> str:
+    hits = [t for t in thresholds if t.get("query") == query]
+    if not hits:
+        return "N/A"
+    return ", ".join(f"{t.get('threshold')}: {t.get('status')}" for t in hits)
+
+
+def target_table(targets: list[dict[str, Any]]) -> str:
+    rows = ["| Target | Status | Validation errors | Region | Datanode data home |", "| --- | --- | ---: | --- | --- |"]
+    for target in targets:
+        errors = len(target.get("validation_errors") or [])
+        discovered = target.get("discovered") or {}
+        if isinstance(discovered, list):
+            region = ", ".join(str(item.get("region_id")) for item in discovered)
+        else:
+            region = discovered.get("region_id")
+        rows.append(
+            "| {name} | {status} {raw} | {errors} | {region} | `{data}` |".format(
+                name=esc(target.get("name")),
+                status=status_emoji(target.get("status")),
+                raw=esc(target.get("status")),
+                errors=errors,
+                region=esc(region),
+                data=esc(target.get("datanode_data_home") or target.get("data_dir")),
+            )
+        )
+    return "\n".join(rows)
+
+
+def comparison_table(targets: list[dict[str, Any]], thresholds: list[dict[str, Any]]) -> str:
+    if len(targets) < 2:
+        return "No base/candidate measurements found."
+    base = measurement_map(targets[0])
+    candidate = measurement_map(targets[1])
+    names = sorted(set(base) | set(candidate))
+    rows = [
+        "| Query | Base median ms | Base p95 ms | Candidate median ms | Candidate p95 ms | Regression | Threshold |",
+        "| --- | ---: | ---: | ---: | ---: | ---: | --- |",
+    ]
+    for name in names:
+        bm = base.get(name, {})
+        cm = candidate.get(name, {})
+        rows.append(
+            "| {q} | {bm} | {bp} | {cm} | {cp} | {reg} | {th} |".format(
+                q=esc(name),
+                bm=fmt_ms(bm.get("latency_ms_median")),
+                bp=fmt_ms(bm.get("latency_ms_p95")),
+                cm=fmt_ms(cm.get("latency_ms_median")),
+                cp=fmt_ms(cm.get("latency_ms_p95")),
+                reg=regression_pct(bm.get("latency_ms_median"), cm.get("latency_ms_median")),
+                th=esc(threshold_status(thresholds, name)),
+            )
+        )
+    return "\n".join(rows)
+
+
+def build_markdown(
+    report: dict[str, Any],
+    report_path: Path,
+    run_url: str | None,
+    case_name: str | None,
+    base_ref: str | None,
+    candidate_ref: str | None,
+) -> str:
+    status = report.get("status", "missing")
+    case = report.get("case") or {}
+    title_case = case_name or case.get("name") or report_path.name
+    lines = [f"## {status_emoji(status)} Query regression report: `{esc(title_case)}`", ""]
+    lines.append(f"- **Status:** `{esc(status)}`")
+    lines.append(f"- **Case path:** `{esc(report.get('case_path'))}`")
+    lines.append(f"- **Query mode:** `{esc(report.get('query_mode'))}`")
+    if base_ref:
+        lines.append(f"- **Base ref:** `{esc(base_ref)}`")
+    if candidate_ref:
+        lines.append(f"- **Candidate ref:** `{esc(candidate_ref)}`")
+    if run_url:
+        lines.append(f"- **Workflow run:** {run_url}")
+    if report.get("error"):
+        lines.append(f"- **Error:** `{esc(report.get('error'))}`")
+    lines.append("- **Artifacts:** query-regression-work logs, fixture metadata, and JSON report are uploaded with this run.")
+
+    targets = report.get("targets") or []
+    lines.extend(["", "### Targets", "", target_table(targets)])
+    lines.extend(["", "### Query comparison", "", comparison_table(targets, report.get("thresholds") or [])])
+
+    not_enforced = [t for t in report.get("thresholds") or [] if t.get("status") == "not_enforced"]
+    if not_enforced:
+        lines.extend(["", "### Not enforced thresholds", ""])
+        for item in not_enforced:
+            lines.append(f"- `{esc(item.get('query'))}` / `{esc(item.get('threshold'))}`: {esc(item.get('reason'))}")
+
+    return "\n".join(lines) + "\n"
+
+
+def build_combined_markdown(
+    reports: list[tuple[Path, dict[str, Any]]],
+    run_url: str | None,
+    case_name: str | None,
+    base_ref: str | None,
+    candidate_ref: str | None,
+) -> str:
+    if len(reports) == 1:
+        path, report = reports[0]
+        return build_markdown(report, path, run_url, case_name, base_ref, candidate_ref)
+
+    overall = "failed" if any(report.get("status") != "ok" for _, report in reports) else "ok"
+    lines = [f"## {status_emoji(overall)} Query regression report: `{esc(case_name or 'multiple cases')}`", ""]
+    lines.append(f"- **Status:** `{esc(overall)}`")
+    if base_ref:
+        lines.append(f"- **Base ref:** `{esc(base_ref)}`")
+    if candidate_ref:
+        lines.append(f"- **Candidate ref:** `{esc(candidate_ref)}`")
+    if run_url:
+        lines.append(f"- **Workflow run:** {run_url}")
+    lines.append("- **Artifacts:** query-regression-work logs, fixture metadata, and JSON reports are uploaded with this run.")
+
+    lines.extend(["", "### Cases", "", "| Case | Status | Report |", "| --- | --- | --- |"])
+    for path, report in reports:
+        case = report.get("case") or {}
+        name = case.get("name") or path.parent.name
+        status = report.get("status", "missing")
+        lines.append(f"| `{esc(name)}` | {status_emoji(status)} `{esc(status)}` | `{esc(path)}` |")
+
+    for path, report in reports:
+        case = report.get("case") or {}
+        name = case.get("name") or path.parent.name
+        lines.extend(["", "---", ""])
+        lines.append(build_markdown(report, path, None, name, None, None).rstrip())
+    return "\n".join(lines) + "\n"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--report", required=True, type=Path, action="append")
+    parser.add_argument("--run-url")
+    parser.add_argument("--case-name")
+    parser.add_argument("--base-ref")
+    parser.add_argument("--candidate-ref")
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+
+    reports = []
+    for report_path in args.report:
+        if report_path.exists():
+            report = json.loads(report_path.read_text())
+        else:
+            report = {"status": "failed", "error": f"report not found: {report_path}", "targets": []}
+        reports.append((report_path, report))
+    markdown = build_combined_markdown(
+        reports,
+        args.run_url,
+        args.case_name,
+        args.base_ref,
+        args.candidate_ref,
+    )
+    if args.output:
+        args.output.write_text(markdown)
+    else:
+        print(markdown, end="")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())