From d1ef863cabe6445bea1d046b413003c93e64abb8 Mon Sep 17 00:00:00 2001 From: shuiyisong Date: Fri, 24 Jul 2026 14:51:50 +0800 Subject: [PATCH] chore: add script to draw result diagram Signed-off-by: shuiyisong --- tests/perf/README.md | 22 ++--- tests/perf/plot_otlp_trace_report.sh | 80 +++++++++++++++++++ tests/perf/query_regression_runner.py | 16 +++- ...ery_regression_runner_compaction_toctou.py | 18 +++++ ...query_regression_runner_otlp_trace_load.py | 67 ++++++++++++++++ 5 files changed, 184 insertions(+), 19 deletions(-) create mode 100755 tests/perf/plot_otlp_trace_report.sh diff --git a/tests/perf/README.md b/tests/perf/README.md index 37e1ee33d35..ce994b6f523 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -222,31 +222,23 @@ candidate `query_perf_fixture`, then run the case explicitly: ```bash WORK_DIR="$(mktemp -d /tmp/query-perf-otlp.XXXXXX)" +REPORT="$WORK_DIR/trace-report.json" python3 tests/perf/query_regression_runner.py \ --case tests/perf/query_cases/otlp_trace_load/case.toml \ --base-bin /path/to/base/target/nightly/greptime \ --candidate-bin /path/to/candidate/target/nightly/greptime \ --fixture-generator /path/to/candidate/target/nightly/query_perf_fixture \ --otelgen-bin /path/to/otelgen \ - --work-dir "$WORK_DIR" + --work-dir "$WORK_DIR" \ + --output "$REPORT" ``` -After the run, print the relevant result fields with `jq`: +Install [YouPlot](https://github.com/red-data-tools/YouPlot) once, then render +all comparison metrics and threshold results in the terminal: ```bash -REPORT="$WORK_DIR/query-regression-report.json" -jq '{ - status, - targets: [.targets[] | { - target: .name, - accepted_spans: .metrics.accepted_spans, - table_rows: .visibility.observed_rows, - spans_per_second: .metrics.accepted_spans_per_second, - mean_latency_ms: .metrics.mean_http_latency_ms, - failures: .metrics.failure_count - }], - thresholds -}' "$REPORT" +brew install youplot +tests/perf/plot_otlp_trace_report.sh "$REPORT" ``` For each target, `accepted_spans` should equal `table_rows`, and `failures` diff --git a/tests/perf/plot_otlp_trace_report.sh b/tests/perf/plot_otlp_trace_report.sh new file mode 100755 index 00000000000..fa1e5ed796d --- /dev/null +++ b/tests/perf/plot_otlp_trace_report.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + printf 'Usage: %s REPORT.json\n' "${0##*/}" >&2 + exit 2 +fi + +report="$1" + +[[ -f "${report}" ]] || { + printf 'Report does not exist: %s\n' "${report}" >&2 + exit 1 +} +command -v jq >/dev/null || { + printf 'jq is required\n' >&2 + exit 1 +} +command -v uplot >/dev/null || { + printf 'YouPlot is required; install it with: brew install youplot\n' >&2 + exit 1 +} + +jq -e ' + (.status | type == "string") and + (.targets | type == "array" and length > 0) and + all(.targets[]; + (.name | type == "string") and + (.metrics.accepted_spans | type == "number") and + (.visibility.observed_rows | type == "number") and + (.metrics.accepted_spans_per_second | type == "number") and + (.metrics.mean_http_latency_ms | type == "number") and + (.metrics.failure_count | type == "number") + ) +' "${report}" >/dev/null || { + printf 'Report is not a completed OTLP trace comparison: %s\n' "${report}" >&2 + exit 1 +} + +plot_metric() { + local title="$1" + local filter="$2" + local color="$3" + + printf '\n' + jq -r ".targets[] | [.name, ${filter}] | @tsv" "${report}" | + uplot bar -t "${title}" -w 80 -c "${color}" -o - +} + +jq -r '"Status: \(.status)"' "${report}" +row_count_status=0 +jq -r ' + "\nAccepted spans vs visible table rows", + (.targets[] | + " [\(if .metrics.accepted_spans == .visibility.observed_rows then "passed" else "failed" end)] \(.name): accepted=\(.metrics.accepted_spans), visible=\(.visibility.observed_rows)" + ) +' "${report}" +jq -e 'all(.targets[]; .metrics.accepted_spans == .visibility.observed_rows)' "${report}" >/dev/null || row_count_status=1 + +plot_metric "Accepted spans" ".metrics.accepted_spans" blue +plot_metric "Visible table rows" ".visibility.observed_rows" green +plot_metric "Throughput (spans/s, higher is better)" ".metrics.accepted_spans_per_second" cyan +plot_metric "Mean HTTP latency (ms, lower is better)" ".metrics.mean_http_latency_ms" yellow +plot_metric "Failures (lower is better)" ".metrics.failure_count" red + +jq -r ' + "\nThresholds", + (.thresholds[] | + if has("actual_pct") then + " [\(.status)] \(.threshold): \(.actual_pct)% (limit \(.limit_pct)%; base \(.base), candidate \(.candidate))" + elif has("actual") then + " [\(.status)] \(.threshold) [\(.target)]: \(.actual) (limit \(.limit))" + else + " [\(.status)] \(.threshold): \(.reason // "no measured value")" + end + ) +' "${report}" + +exit "${row_count_status}" diff --git a/tests/perf/query_regression_runner.py b/tests/perf/query_regression_runner.py index 3ea9542509c..724c38ebf17 100644 --- a/tests/perf/query_regression_runner.py +++ b/tests/perf/query_regression_runner.py @@ -78,6 +78,7 @@ def parse_args() -> argparse.Namespace: p.add_argument("--remote-write-generator", type=Path, help="deprecated: use --fixture-generator query_perf_fixture") p.add_argument("--storage-inspector", type=Path, help="deprecated: use --fixture-generator query_perf_fixture") p.add_argument("--work-dir", required=True, type=Path) + p.add_argument("--output", type=Path, help="write the final JSON report to this file instead of stdout") p.add_argument("--fixture-cache-dir", type=Path, help="persistent directory for generated fixtures, keyed by case content") p.add_argument("--reuse-fixture", action="store_true") p.add_argument("--allow-large-fixture", action="store_true") @@ -1041,6 +1042,13 @@ def write_json(path: Path, payload: dict[str, Any]) -> None: path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") +def output_report(report: dict[str, Any], output: Path | None) -> None: + if output is None: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + write_json(output, report) + + def parse_prometheus_metrics(text: str, names: set[str] = OTLP_TRACE_METRICS) -> dict[str, float]: values: dict[str, float] = {} for line in text.splitlines(): @@ -1554,7 +1562,7 @@ def main() -> int: report["status"] = "failed" report["error"] = repr(e) write_json(work_root / "query-regression-report.json", report) - print(json.dumps(report, indent=2, sort_keys=True)) + output_report(report, args.output) return 1 if report["status"] == "failed" else 0 if scenario_kind == "prom_remote_write_then_query": @@ -1564,7 +1572,7 @@ def main() -> int: report["status"] = "failed" report["error"] = repr(e) write_json(work_root / "query-regression-report.json", report) - print(json.dumps(report, indent=2, sort_keys=True)) + output_report(report, args.output) return 1 if report["status"] == "failed" else 0 if args.fixture_only or args.dry_run: @@ -1584,7 +1592,7 @@ def main() -> int: report["targets"].append(tr) report["status"] = "planned" if args.dry_run else "fixture-ready" write_json(work_root / "query-regression-report.json", report) - print(json.dumps(report, indent=2, sort_keys=True)) + output_report(report, args.output) return 0 clusters: list[DistributedCluster] = [] @@ -1647,7 +1655,7 @@ def main() -> int: for cluster in reversed(clusters): cluster.stop_all() write_json(work_root / "query-regression-report.json", report) - print(json.dumps(report, indent=2, sort_keys=True)) + output_report(report, args.output) return 1 if report["status"] == "failed" else 0 diff --git a/tests/perf/test_query_regression_runner_compaction_toctou.py b/tests/perf/test_query_regression_runner_compaction_toctou.py index 675f451a10e..0e79cf245de 100644 --- a/tests/perf/test_query_regression_runner_compaction_toctou.py +++ b/tests/perf/test_query_regression_runner_compaction_toctou.py @@ -17,6 +17,7 @@ import argparse import importlib.util +import json import sys import tempfile import unittest @@ -32,6 +33,23 @@ sys.modules[SPEC.name] = runner SPEC.loader.exec_module(runner) +class ReportOutputTest(unittest.TestCase): + def test_final_report_output_defaults_to_stdout_or_writes_to_file(self) -> None: + report = {"status": "ok"} + expected = json.dumps(report, indent=2, sort_keys=True) + + with patch("builtins.print") as print_mock: + runner.output_report(report, None) + print_mock.assert_called_once_with(expected) + + with tempfile.TemporaryDirectory() as tmpdir: + output = Path(tmpdir) / "nested" / "report.json" + with patch("builtins.print") as print_mock: + runner.output_report(report, output) + print_mock.assert_not_called() + self.assertEqual(output.read_text(), expected + "\n") + + class RemoteWriteCompactionToctouTest(unittest.TestCase): def test_quiesces_each_target_before_storage_inspection_and_read_bench(self) -> None: lifecycle: dict[str, list[str]] = {"base": [], "candidate": []} diff --git a/tests/perf/test_query_regression_runner_otlp_trace_load.py b/tests/perf/test_query_regression_runner_otlp_trace_load.py index 9b2a15d8924..ab14ceef24c 100644 --- a/tests/perf/test_query_regression_runner_otlp_trace_load.py +++ b/tests/perf/test_query_regression_runner_otlp_trace_load.py @@ -16,6 +16,8 @@ """Regression coverage for the local OTLP trace load lifecycle and metrics.""" import importlib.util +import json +import os import subprocess import sys import tempfile @@ -25,6 +27,7 @@ from unittest.mock import patch RUNNER_PATH = Path(__file__).with_name("query_regression_runner.py") +PLOTTER_PATH = Path(__file__).with_name("plot_otlp_trace_report.sh") SPEC = importlib.util.spec_from_file_location("query_regression_runner_otlp_under_test", RUNNER_PATH) assert SPEC is not None and SPEC.loader is not None runner = importlib.util.module_from_spec(SPEC) @@ -32,6 +35,70 @@ sys.modules[SPEC.name] = runner SPEC.loader.exec_module(runner) +class OtlpTraceReportPlotTest(unittest.TestCase): + def test_plots_all_comparison_metrics_and_thresholds(self) -> None: + report = { + "status": "ok", + "targets": [ + { + "name": name, + "metrics": { + "accepted_spans": spans, + "accepted_spans_per_second": rate, + "mean_http_latency_ms": latency, + "failure_count": 0, + }, + "visibility": {"observed_rows": spans}, + } + for name, spans, rate, latency in (("base", 100, 10.0, 3.0), ("candidate", 120, 12.0, 2.5)) + ], + "thresholds": [ + {"target": "base", "threshold": "max_failure_count", "status": "passed", "actual": 0, "limit": 0}, + { + "threshold": "max_candidate_throughput_regression_pct", + "status": "passed", + "actual_pct": -20.0, + "limit_pct": 20.0, + "base": 10.0, + "candidate": 12.0, + }, + ], + } + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + report_path = root / "report.json" + report_path.write_text(json.dumps(report)) + fake_bin = root / "bin" + fake_bin.mkdir() + uplot = fake_bin / "uplot" + uplot.write_text('#!/usr/bin/env bash\nprintf "CALL %s\\n" "$*" >>"$UPLOT_LOG"\ncat >>"$UPLOT_LOG"\n') + uplot.chmod(0o755) + log_path = root / "uplot.log" + env = os.environ.copy() + env["PATH"] = f"{fake_bin}{os.pathsep}{env['PATH']}" + env["UPLOT_LOG"] = str(log_path) + + result = subprocess.run([str(PLOTTER_PATH), str(report_path)], env=env, text=True, capture_output=True, check=False) + log = log_path.read_text() + report["targets"][1]["visibility"]["observed_rows"] = 119 + report_path.write_text(json.dumps(report)) + mismatch = subprocess.run([str(PLOTTER_PATH), str(report_path)], env=env, text=True, capture_output=True, check=False) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(log.count("CALL "), 5) + for metric in ("Accepted spans", "Visible table rows", "Throughput", "Mean HTTP latency", "Failures"): + self.assertIn(metric, log) + self.assertIn("base\t100", log) + self.assertIn("candidate\t12", log) + self.assertIn("Status: ok", result.stdout) + self.assertIn("[passed] candidate: accepted=120, visible=120", result.stdout) + self.assertIn("max_failure_count", result.stdout) + self.assertIn("max_candidate_throughput_regression_pct", result.stdout) + self.assertEqual(mismatch.returncode, 1, mismatch.stderr) + self.assertIn("[failed] candidate: accepted=120, visible=119", mismatch.stdout) + + class OtlpTraceLoadTest(unittest.TestCase): def test_stops_base_cluster_before_creating_candidate_cluster(self) -> None: events = []