diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index e86ef707734..2dcec403f68 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -302,6 +302,11 @@ CPU_PROFILE_SECONDS = 10 CPU_PROFILE_FREQUENCY = 49 CPU_PROFILE_TIMEOUT_SECONDS = 180 CPU_PROFILE_MANIFEST = Path("logs/cpu-profile-manifest.json") +POSTMEASUREMENT_ANALYZE_MANIFEST = Path("logs/postmeasurement-analyze-manifest.json") +POSTMEASUREMENT_ANALYZE_CASES = { + "promql_constant_tag_concat", + "promql_constant_tag_concat_ms", +} def select_cpu_profile_query(plan: dict[str, Any]) -> tuple[dict[str, str] | None, str | None]: @@ -388,11 +393,12 @@ def query_response_error(payload: Any) -> str | None: return None -def post_profile_query( +def post_query( target: RunTarget, query: dict[str, str], timeout_s: float, -) -> tuple[bool, str | None]: +) -> tuple[bool, str | None, Any | None]: + """Execute a SQL endpoint query and retain its complete JSON response.""" body = urllib.parse.urlencode( {"sql": query["query"], "db": query["database"], "format": "json"}, ).encode() @@ -405,11 +411,22 @@ def post_profile_query( with urllib.request.urlopen(request, timeout=timeout_s) as response: payload = response.read() if response.status >= 400: - return False, f"HTTP {response.status}" - error = query_response_error(json.loads(payload)) - return error is None, error - except Exception as err: # noqa: BLE001 - request diagnostics belong in the profile manifest - return False, repr(err) + return False, f"HTTP {response.status}", None + response_json = json.loads(payload) + error = query_response_error(response_json) + return error is None, error, response_json + except Exception as err: # noqa: BLE001 - request diagnostics belong in runner artifacts + return False, repr(err), None + + +def post_profile_query( + target: RunTarget, + query: dict[str, str], + timeout_s: float, +) -> tuple[bool, str | None]: + """Execute the selected ordinary query while CPU profiling.""" + ok, error, _response = post_query(target, query, timeout_s) + return ok, error def download_cpu_profile(port: int, output: Path) -> None: @@ -548,6 +565,79 @@ def collect_cpu_profiles(args: argparse.Namespace, plan: dict[str, Any], work_di print(f"CPU profile {manifest['status']}: {reason}; manifest: {manifest_path}", flush=True) return manifest["status"] + +def collect_postmeasurement_analyze( + case_path: Path, + plan: dict[str, Any], + work_dir: Path, + targets: list[RunTarget], + http_timeout: float, +) -> str: + """Save ANALYZE VERBOSE diagnostics after ordinary-query measurement and profiling.""" + manifest: dict[str, Any] = { + "status": "not_applicable", + "targets": [], + "errors": [], + } + if case_slug(case_path) not in POSTMEASUREMENT_ANALYZE_CASES: + return manifest["status"] + + selected, reason = select_cpu_profile_query(plan) + scenario = plan.get("scenario") + configured_queries = scenario.get("queries") if isinstance(scenario, dict) else None + diagnostics: list[dict[str, str]] = [] + if selected is not None and isinstance(configured_queries, list): + for query in configured_queries: + if not isinstance(query, dict) or query.get("kind") != "tql": + reason = f"postmeasurement query is not TQL: {query!r}" + diagnostics = [] + break + statement = query.get("query") + if not isinstance(statement, str) or not statement.startswith("TQL EVAL "): + reason = f"postmeasurement query is not an ordinary TQL EVAL query: {statement!r}" + break + name = query.get("name") + diagnostics.append({ + "name": name if isinstance(name, str) else statement, + "database": selected["database"], + "source_query": statement, + "query": statement.replace("TQL EVAL ", "TQL ANALYZE VERBOSE ", 1), + }) + if selected is None or reason is not None or not diagnostics: + manifest["status"] = "failure" + manifest["errors"].append(reason or "no TQL queries are configured for postmeasurement analysis") + else: + for target in targets: + target_manifest: dict[str, Any] = {"role": target.name, "queries": []} + manifest["targets"].append(target_manifest) + for diagnostic in diagnostics: + ok, error, response = post_query(target, diagnostic, http_timeout) + result: dict[str, Any] = { + "name": diagnostic["name"], + "source_query": diagnostic["source_query"], + "query": diagnostic["query"], + "status": "success" if ok else "failure", + } + if response is not None: + # Keep the full server result; these diagnostics are not benchmark samples. + result["response"] = response + if not ok: + result["error"] = error + manifest["errors"].append(f"{target.name}/{diagnostic['name']}: {error}") + target_manifest["queries"].append(result) + manifest["status"] = "failure" if manifest["errors"] else "success" + + manifest_path = work_dir / POSTMEASUREMENT_ANALYZE_MANIFEST + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if manifest["status"] != "success": + print( + f"Postmeasurement ANALYZE {manifest['status']}: {manifest['errors']}; manifest: {manifest_path}", + flush=True, + ) + return manifest["status"] + + def run_direct_case( args: argparse.Namespace, case_path: Path, @@ -634,7 +724,11 @@ def run_direct_case( status = subprocess.run(measure, check=False).returncode if status != 0: return status - return 0 if collect_cpu_profiles(args, plan, work_dir, targets) == "success" else 1 + profile_status = collect_cpu_profiles(args, plan, work_dir, targets) + analyze_status = collect_postmeasurement_analyze( + case_path, plan, work_dir, targets, float(args.http_timeout), + ) + return 0 if profile_status == "success" and analyze_status in {"success", "not_applicable"} else 1 finally: stop_all(targets, procs) diff --git a/tests/perf/query_cases/promql_constant_tag_concat/case.toml b/tests/perf/query_cases/promql_constant_tag_concat/case.toml index f876d1d871a..e28f771ead3 100644 --- a/tests/perf/query_cases/promql_constant_tag_concat/case.toml +++ b/tests/perf/query_cases/promql_constant_tag_concat/case.toml @@ -60,7 +60,7 @@ series_layout = "timestamp_major" [[scenario.queries]] name = "selector_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') promql_constant_tag_concat" +query = "TQL EVAL (1704069000, 1704069900, '15s') promql_constant_tag_concat" warmup = 3 iterations = 9 @@ -70,7 +70,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "selector_host1" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') promql_constant_tag_concat{host='host1'}" +query = "TQL EVAL (1704069000, 1704069900, '15s') promql_constant_tag_concat{host='host1'}" warmup = 3 iterations = 9 @@ -80,7 +80,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "topk_5_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat)" +query = "TQL EVAL (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat)" warmup = 3 iterations = 9 @@ -90,7 +90,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "topk_5_host1" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat{host='host1'})" +query = "TQL EVAL (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat{host='host1'})" warmup = 3 iterations = 9 @@ -100,7 +100,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "topk_100_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') topk(100, promql_constant_tag_concat)" +query = "TQL EVAL (1704069000, 1704069900, '15s') topk(100, promql_constant_tag_concat)" warmup = 3 iterations = 9 @@ -110,7 +110,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "bottomk_5_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') bottomk(5, promql_constant_tag_concat)" +query = "TQL EVAL (1704069000, 1704069900, '15s') bottomk(5, promql_constant_tag_concat)" warmup = 3 iterations = 9 @@ -122,7 +122,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "instant_selector_single_evaluation_control" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069000, '15s') promql_constant_tag_concat" +query = "TQL EVAL (1704069000, 1704069000, '15s') promql_constant_tag_concat" warmup = 3 iterations = 9 diff --git a/tests/perf/query_cases/promql_constant_tag_concat_ms/case.toml b/tests/perf/query_cases/promql_constant_tag_concat_ms/case.toml index 3a2bfc66283..0a4f7114753 100644 --- a/tests/perf/query_cases/promql_constant_tag_concat_ms/case.toml +++ b/tests/perf/query_cases/promql_constant_tag_concat_ms/case.toml @@ -62,7 +62,7 @@ series_layout = "timestamp_major" [[scenario.queries]] name = "selector_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') promql_constant_tag_concat_ms" +query = "TQL EVAL (1704069000, 1704069900, '15s') promql_constant_tag_concat_ms" warmup = 3 iterations = 9 @@ -72,7 +72,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "selector_host1" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') promql_constant_tag_concat_ms{host='host1'}" +query = "TQL EVAL (1704069000, 1704069900, '15s') promql_constant_tag_concat_ms{host='host1'}" warmup = 3 iterations = 9 @@ -82,7 +82,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "topk_5_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat_ms)" +query = "TQL EVAL (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat_ms)" warmup = 3 iterations = 9 @@ -92,7 +92,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "topk_5_host1" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat_ms{host='host1'})" +query = "TQL EVAL (1704069000, 1704069900, '15s') topk(5, promql_constant_tag_concat_ms{host='host1'})" warmup = 3 iterations = 9 @@ -102,7 +102,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "topk_100_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') topk(100, promql_constant_tag_concat_ms)" +query = "TQL EVAL (1704069000, 1704069900, '15s') topk(100, promql_constant_tag_concat_ms)" warmup = 3 iterations = 9 @@ -112,7 +112,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "bottomk_5_full" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069900, '15s') bottomk(5, promql_constant_tag_concat_ms)" +query = "TQL EVAL (1704069000, 1704069900, '15s') bottomk(5, promql_constant_tag_concat_ms)" warmup = 3 iterations = 9 @@ -124,7 +124,7 @@ max_candidate_latency_regression_pct = 10 [[scenario.queries]] name = "instant_selector_single_evaluation_control" kind = "tql" -query = "TQL ANALYZE VERBOSE (1704069000, 1704069000, '15s') promql_constant_tag_concat_ms" +query = "TQL EVAL (1704069000, 1704069000, '15s') promql_constant_tag_concat_ms" warmup = 3 iterations = 9 diff --git a/tests/perf/test_query_regression_case_selection.py b/tests/perf/test_query_regression_case_selection.py index 706d0f67909..e250ccd6fc5 100644 --- a/tests/perf/test_query_regression_case_selection.py +++ b/tests/perf/test_query_regression_case_selection.py @@ -150,6 +150,108 @@ class QueryRegressionCpuProfileTest(unittest.TestCase): self.assertEqual("chosen", manifest["query"]["selected"]["name"]) self.assertEqual(["base", "candidate"], [target["role"] for target in manifest["targets"]]) + def test_constant_tag_concat_cases_time_and_profile_ordinary_tql_eval(self) -> None: + for name in ("promql_constant_tag_concat", "promql_constant_tag_concat_ms"): + case_path = Path(__file__).parent / "query_cases" / name / "case.toml" + queries = [ + line.removeprefix('query = "').removesuffix('"') + for line in case_path.read_text().splitlines() + if line.startswith('query = "') + ] + self.assertTrue(queries) + self.assertTrue(all(query.startswith("TQL EVAL ") for query in queries)) + + plan = {"scenario": {"kind": "direct_readable_sst", "tables": [{"database": "public"}], "queries": [ + {"kind": "tql", "query": query, "iterations": 1} for query in queries + ]}} + profile_query, reason = runner.select_cpu_profile_query(plan) + self.assertIsNone(reason) + self.assertTrue(profile_query["query"].startswith("TQL EVAL ")) + + def test_constant_tag_concat_analyze_is_postmeasurement_and_has_its_own_full_response_artifact(self) -> None: + from tempfile import TemporaryDirectory + from types import SimpleNamespace + from unittest.mock import patch + + with TemporaryDirectory() as directory: + root = Path(directory) + work_dir = root / "work" + case_path = root / "promql_constant_tag_concat" / "case.toml" + events: list[str] = [] + plan = { + "scenario": { + "kind": "direct_readable_sst", + "tables": [{"database": "public"}], + "queries": [{"name": "selected", "kind": "tql", "query": "TQL EVAL (1, 2, '1s') metric", "iterations": 1}], + }, + } + + def run(command: list[str], **_kwargs: object) -> object: + if "prepare-direct" in command: + Path(command[command.index("--output") + 1]).write_text(json.dumps({"fixtures": []})) + if "measure" in command: + events.append("measure") + Path(command[command.index("--output") + 1]).write_text(json.dumps({"samples": ["ordinary"]})) + return SimpleNamespace(returncode=0) + + def profile(*_args: object, **_kwargs: object) -> str: + events.append("profile") + return "success" + + def post(_target: object, query: dict[str, str], _timeout: float) -> tuple[bool, None, dict[str, object]]: + events.append("analyze") + self.assertEqual("TQL ANALYZE VERBOSE (1, 2, '1s') metric", query["query"]) + return True, None, {"output": [{"rows": [["all diagnostic rows"]]}]} + + def stop(*_args: object) -> None: + events.append("cleanup") + + with ( + patch.object(runner, "allocate_ports", return_value=list(range(1000, 1016))), + patch.object(runner, "start_component"), + patch.object(runner, "stop_component", side_effect=stop), + patch.object(runner.subprocess, "run", side_effect=run), + patch.object(runner, "collect_cpu_profiles", side_effect=profile), + patch.object(runner, "post_query", side_effect=post), + ): + status = runner.run_direct_case( + SimpleNamespace(http_timeout="1", allow_large_fixture="false"), + case_path, work_dir, Path("base"), Path("candidate"), Path("fixture"), Path("runner"), plan, + ) + + self.assertEqual(0, status) + self.assertLess(events.index("measure"), events.index("profile")) + self.assertLess(events.index("profile"), events.index("analyze")) + self.assertLess(events.index("analyze"), len(events) - 1 - events[::-1].index("cleanup")) + self.assertEqual(["ordinary"], json.loads((work_dir / "query-regression-report.json").read_text())["samples"]) + manifest = json.loads((work_dir / runner.POSTMEASUREMENT_ANALYZE_MANIFEST).read_text()) + self.assertEqual("success", manifest["status"]) + self.assertEqual({"base", "candidate"}, {target["role"] for target in manifest["targets"]}) + self.assertEqual( + {"output": [{"rows": [["all diagnostic rows"]]}]}, + manifest["targets"][0]["queries"][0]["response"], + ) + + def test_constant_tag_concat_analyze_failure_is_reported_and_fails_the_case(self) -> None: + from tempfile import TemporaryDirectory + from unittest.mock import patch + + with TemporaryDirectory() as directory: + root = Path(directory) + target = self.make_target("base", root, 1000) + plan = {"scenario": {"kind": "direct_readable_sst", "tables": [{"database": "public"}], "queries": [ + {"name": "selected", "kind": "tql", "query": "TQL EVAL (1, 2, '1s') metric", "iterations": 1}, + ]}} + with patch.object(runner, "post_query", return_value=(False, "HTTP 500", None)): + status = runner.collect_postmeasurement_analyze( + root / "promql_constant_tag_concat" / "case.toml", plan, root, [target], 1.0, + ) + + self.assertEqual("failure", status) + manifest = json.loads((root / runner.POSTMEASUREMENT_ANALYZE_MANIFEST).read_text()) + self.assertEqual("failure", manifest["status"]) + self.assertIn("HTTP 500", manifest["errors"][0]) + def test_profile_query_response_errors_are_failures(self) -> None: self.assertIsNone(runner.query_response_error({"code": 0})) self.assertEqual("query response error: denied", runner.query_response_error({"error": "denied"}))