chore: add script to draw result diagram

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

tests/perf/README.md

@@ -222,31 +222,23 @@ candidate `query_perf_fixture`, then run the case explicitly:
 
 ```bash
 WORK_DIR="$(mktemp -d /tmp/query-perf-otlp.XXXXXX)"
+REPORT="$WORK_DIR/trace-report.json"
 python3 tests/perf/query_regression_runner.py \
   --case tests/perf/query_cases/otlp_trace_load/case.toml \
   --base-bin /path/to/base/target/nightly/greptime \
   --candidate-bin /path/to/candidate/target/nightly/greptime \
   --fixture-generator /path/to/candidate/target/nightly/query_perf_fixture \
   --otelgen-bin /path/to/otelgen \
-  --work-dir "$WORK_DIR"
+  --work-dir "$WORK_DIR" \
+  --output "$REPORT"
 ```
 
-After the run, print the relevant result fields with `jq`:
+Install [YouPlot](https://github.com/red-data-tools/YouPlot) once, then render
+all comparison metrics and threshold results in the terminal:
 
 ```bash
-REPORT="$WORK_DIR/query-regression-report.json"
-jq '{
-  status,
-  targets: [.targets[] | {
-    target: .name,
-    accepted_spans: .metrics.accepted_spans,
-    table_rows: .visibility.observed_rows,
-    spans_per_second: .metrics.accepted_spans_per_second,
-    mean_latency_ms: .metrics.mean_http_latency_ms,
-    failures: .metrics.failure_count
-  }],
-  thresholds
-}' "$REPORT"
+brew install youplot
+tests/perf/plot_otlp_trace_report.sh "$REPORT"
 ```
 
 For each target, `accepted_spans` should equal `table_rows`, and `failures`

tests/perf/plot_otlp_trace_report.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  printf 'Usage: %s REPORT.json\n' "${0##*/}" >&2
+  exit 2
+fi
+
+report="$1"
+
+[[ -f "${report}" ]] || {
+  printf 'Report does not exist: %s\n' "${report}" >&2
+  exit 1
+}
+command -v jq >/dev/null || {
+  printf 'jq is required\n' >&2
+  exit 1
+}
+command -v uplot >/dev/null || {
+  printf 'YouPlot is required; install it with: brew install youplot\n' >&2
+  exit 1
+}
+
+jq -e '
+  (.status | type == "string") and
+  (.targets | type == "array" and length > 0) and
+  all(.targets[];
+    (.name | type == "string") and
+    (.metrics.accepted_spans | type == "number") and
+    (.visibility.observed_rows | type == "number") and
+    (.metrics.accepted_spans_per_second | type == "number") and
+    (.metrics.mean_http_latency_ms | type == "number") and
+    (.metrics.failure_count | type == "number")
+  )
+' "${report}" >/dev/null || {
+  printf 'Report is not a completed OTLP trace comparison: %s\n' "${report}" >&2
+  exit 1
+}
+
+plot_metric() {
+  local title="$1"
+  local filter="$2"
+  local color="$3"
+
+  printf '\n'
+  jq -r ".targets[] | [.name, ${filter}] | @tsv" "${report}" |
+    uplot bar -t "${title}" -w 80 -c "${color}" -o -
+}
+
+jq -r '"Status: \(.status)"' "${report}"
+row_count_status=0
+jq -r '
+  "\nAccepted spans vs visible table rows",
+  (.targets[] |
+    "  [\(if .metrics.accepted_spans == .visibility.observed_rows then "passed" else "failed" end)] \(.name): accepted=\(.metrics.accepted_spans), visible=\(.visibility.observed_rows)"
+  )
+' "${report}"
+jq -e 'all(.targets[]; .metrics.accepted_spans == .visibility.observed_rows)' "${report}" >/dev/null || row_count_status=1
+
+plot_metric "Accepted spans" ".metrics.accepted_spans" blue
+plot_metric "Visible table rows" ".visibility.observed_rows" green
+plot_metric "Throughput (spans/s, higher is better)" ".metrics.accepted_spans_per_second" cyan
+plot_metric "Mean HTTP latency (ms, lower is better)" ".metrics.mean_http_latency_ms" yellow
+plot_metric "Failures (lower is better)" ".metrics.failure_count" red
+
+jq -r '
+  "\nThresholds",
+  (.thresholds[] |
+    if has("actual_pct") then
+      "  [\(.status)] \(.threshold): \(.actual_pct)% (limit \(.limit_pct)%; base \(.base), candidate \(.candidate))"
+    elif has("actual") then
+      "  [\(.status)] \(.threshold) [\(.target)]: \(.actual) (limit \(.limit))"
+    else
+      "  [\(.status)] \(.threshold): \(.reason // "no measured value")"
+    end
+  )
+' "${report}"
+
+exit "${row_count_status}"

tests/perf/query_regression_runner.py

@@ -78,6 +78,7 @@ def parse_args() -> argparse.Namespace:
     p.add_argument("--remote-write-generator", type=Path, help="deprecated: use --fixture-generator query_perf_fixture")
     p.add_argument("--storage-inspector", type=Path, help="deprecated: use --fixture-generator query_perf_fixture")
     p.add_argument("--work-dir", required=True, type=Path)
+    p.add_argument("--output", type=Path, help="write the final JSON report to this file instead of stdout")
     p.add_argument("--fixture-cache-dir", type=Path, help="persistent directory for generated fixtures, keyed by case content")
     p.add_argument("--reuse-fixture", action="store_true")
     p.add_argument("--allow-large-fixture", action="store_true")
@@ -1041,6 +1042,13 @@ def write_json(path: Path, payload: dict[str, Any]) -> None:
     path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
 
 
+def output_report(report: dict[str, Any], output: Path | None) -> None:
+    if output is None:
+        print(json.dumps(report, indent=2, sort_keys=True))
+    else:
+        write_json(output, report)
+
+
 def parse_prometheus_metrics(text: str, names: set[str] = OTLP_TRACE_METRICS) -> dict[str, float]:
     values: dict[str, float] = {}
     for line in text.splitlines():
@@ -1554,7 +1562,7 @@ def main() -> int:
             report["status"] = "failed"
             report["error"] = repr(e)
         write_json(work_root / "query-regression-report.json", report)
-        print(json.dumps(report, indent=2, sort_keys=True))
+        output_report(report, args.output)
         return 1 if report["status"] == "failed" else 0
 
     if scenario_kind == "prom_remote_write_then_query":
@@ -1564,7 +1572,7 @@ def main() -> int:
             report["status"] = "failed"
             report["error"] = repr(e)
         write_json(work_root / "query-regression-report.json", report)
-        print(json.dumps(report, indent=2, sort_keys=True))
+        output_report(report, args.output)
         return 1 if report["status"] == "failed" else 0
 
     if args.fixture_only or args.dry_run:
@@ -1584,7 +1592,7 @@ def main() -> int:
             report["targets"].append(tr)
         report["status"] = "planned" if args.dry_run else "fixture-ready"
         write_json(work_root / "query-regression-report.json", report)
-        print(json.dumps(report, indent=2, sort_keys=True))
+        output_report(report, args.output)
         return 0
 
     clusters: list[DistributedCluster] = []
@@ -1647,7 +1655,7 @@ def main() -> int:
         for cluster in reversed(clusters):
             cluster.stop_all()
     write_json(work_root / "query-regression-report.json", report)
-    print(json.dumps(report, indent=2, sort_keys=True))
+    output_report(report, args.output)
     return 1 if report["status"] == "failed" else 0
 
 

tests/perf/test_query_regression_runner_compaction_toctou.py

@@ -17,6 +17,7 @@
 
 import argparse
 import importlib.util
+import json
 import sys
 import tempfile
 import unittest
@@ -32,6 +33,23 @@ sys.modules[SPEC.name] = runner
 SPEC.loader.exec_module(runner)
 
 
+class ReportOutputTest(unittest.TestCase):
+    def test_final_report_output_defaults_to_stdout_or_writes_to_file(self) -> None:
+        report = {"status": "ok"}
+        expected = json.dumps(report, indent=2, sort_keys=True)
+
+        with patch("builtins.print") as print_mock:
+            runner.output_report(report, None)
+        print_mock.assert_called_once_with(expected)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output = Path(tmpdir) / "nested" / "report.json"
+            with patch("builtins.print") as print_mock:
+                runner.output_report(report, output)
+            print_mock.assert_not_called()
+            self.assertEqual(output.read_text(), expected + "\n")
+
+
 class RemoteWriteCompactionToctouTest(unittest.TestCase):
     def test_quiesces_each_target_before_storage_inspection_and_read_bench(self) -> None:
         lifecycle: dict[str, list[str]] = {"base": [], "candidate": []}

tests/perf/test_query_regression_runner_otlp_trace_load.py

@@ -16,6 +16,8 @@
 """Regression coverage for the local OTLP trace load lifecycle and metrics."""
 
 import importlib.util
+import json
+import os
 import subprocess
 import sys
 import tempfile
@@ -25,6 +27,7 @@ from unittest.mock import patch
 
 
 RUNNER_PATH = Path(__file__).with_name("query_regression_runner.py")
+PLOTTER_PATH = Path(__file__).with_name("plot_otlp_trace_report.sh")
 SPEC = importlib.util.spec_from_file_location("query_regression_runner_otlp_under_test", RUNNER_PATH)
 assert SPEC is not None and SPEC.loader is not None
 runner = importlib.util.module_from_spec(SPEC)
@@ -32,6 +35,70 @@ sys.modules[SPEC.name] = runner
 SPEC.loader.exec_module(runner)
 
 
+class OtlpTraceReportPlotTest(unittest.TestCase):
+    def test_plots_all_comparison_metrics_and_thresholds(self) -> None:
+        report = {
+            "status": "ok",
+            "targets": [
+                {
+                    "name": name,
+                    "metrics": {
+                        "accepted_spans": spans,
+                        "accepted_spans_per_second": rate,
+                        "mean_http_latency_ms": latency,
+                        "failure_count": 0,
+                    },
+                    "visibility": {"observed_rows": spans},
+                }
+                for name, spans, rate, latency in (("base", 100, 10.0, 3.0), ("candidate", 120, 12.0, 2.5))
+            ],
+            "thresholds": [
+                {"target": "base", "threshold": "max_failure_count", "status": "passed", "actual": 0, "limit": 0},
+                {
+                    "threshold": "max_candidate_throughput_regression_pct",
+                    "status": "passed",
+                    "actual_pct": -20.0,
+                    "limit_pct": 20.0,
+                    "base": 10.0,
+                    "candidate": 12.0,
+                },
+            ],
+        }
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            report_path = root / "report.json"
+            report_path.write_text(json.dumps(report))
+            fake_bin = root / "bin"
+            fake_bin.mkdir()
+            uplot = fake_bin / "uplot"
+            uplot.write_text('#!/usr/bin/env bash\nprintf "CALL %s\\n" "$*" >>"$UPLOT_LOG"\ncat >>"$UPLOT_LOG"\n')
+            uplot.chmod(0o755)
+            log_path = root / "uplot.log"
+            env = os.environ.copy()
+            env["PATH"] = f"{fake_bin}{os.pathsep}{env['PATH']}"
+            env["UPLOT_LOG"] = str(log_path)
+
+            result = subprocess.run([str(PLOTTER_PATH), str(report_path)], env=env, text=True, capture_output=True, check=False)
+            log = log_path.read_text()
+            report["targets"][1]["visibility"]["observed_rows"] = 119
+            report_path.write_text(json.dumps(report))
+            mismatch = subprocess.run([str(PLOTTER_PATH), str(report_path)], env=env, text=True, capture_output=True, check=False)
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(log.count("CALL "), 5)
+        for metric in ("Accepted spans", "Visible table rows", "Throughput", "Mean HTTP latency", "Failures"):
+            self.assertIn(metric, log)
+        self.assertIn("base\t100", log)
+        self.assertIn("candidate\t12", log)
+        self.assertIn("Status: ok", result.stdout)
+        self.assertIn("[passed] candidate: accepted=120, visible=120", result.stdout)
+        self.assertIn("max_failure_count", result.stdout)
+        self.assertIn("max_candidate_throughput_regression_pct", result.stdout)
+        self.assertEqual(mismatch.returncode, 1, mismatch.stderr)
+        self.assertIn("[failed] candidate: accepted=120, visible=119", mismatch.stdout)
+
+
 class OtlpTraceLoadTest(unittest.TestCase):
     def test_stops_base_cluster_before_creating_candidate_cluster(self) -> None:
         events = []