diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index 07593d7b2f5..d5fa1628ebc 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -27,6 +27,7 @@ from pathlib import Path DEFAULT_CASES = [ "tests/perf/query_cases/smoke_direct_sst/case.toml", "tests/perf/query_cases/prom_remote_write_smoke/case.toml", + "tests/perf/query_cases/prom_remote_write_heavy_storage/case.toml", "tests/perf/query_cases/promql_pushdown_7913/case.toml", "tests/perf/query_cases/sql_topk_order_by/case.toml", "tests/perf/query_cases/sql_aggregate_order_by/case.toml", diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index b87d81fa42d..2cfa5035aaf 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -58,7 +58,7 @@ jobs: contains(github.event.pull_request.labels.*.name, 'query-regression') && (github.event.action != 'labeled' || github.event.label.name == 'query-regression')) }} runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || 'perf-regression-8-cores' }} - timeout-minutes: 180 + timeout-minutes: 480 env: CARGO_PROFILE: ${{ github.event_name == 'pull_request' && 'nightly' || inputs.cargo_profile }} CARGO_TARGET_DIR: ${{ github.workspace }}/query-regression-target diff --git a/tests/perf/README.md b/tests/perf/README.md index 62b33b09626..f12b2537f82 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -183,6 +183,13 @@ into three flushed chunks, and requires multiple landed SSTs so storage inspection and the read-bench supplement have enough data to reduce incidental query noise while staying much smaller than the manual 7913 case. +`tests/perf/query_cases/prom_remote_write_heavy_storage/case.toml` is also in +the default self-hosted query-regression case set. It writes 16,384 series × +43,200 samples (30 days at 1-minute resolution), flushes once per day, and +requires at least 20 landed SSTs with `greptime_value`. This heavy case is meant +to make storage/read/query costs visible enough that baseline queries can take +tens of seconds on release-equivalent perf runners. + `tests/perf/query_cases/prom_remote_write_7913/case.toml` is a larger manual case for issue #7913. It writes 8192 series × 20160 samples through remote-write in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d diff --git a/tests/perf/query_cases/prom_remote_write_heavy_storage/case.toml b/tests/perf/query_cases/prom_remote_write_heavy_storage/case.toml new file mode 100644 index 00000000000..c1c70ee2bf8 --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_heavy_storage/case.toml @@ -0,0 +1,68 @@ +# Heavy default Prometheus remote-write storage/read-bench scenario. +# +# This case intentionally exercises the normal metric write path at a size large +# enough to land dozens of SSTs and make base query/read costs visible on the +# self-hosted query-regression runners. + +[case] +name = "prom_remote_write_heavy_storage" +description = "Heavy Prometheus remote-write storage/read-bench case with dozens of flushed SSTs" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_heavy_storage" +physical_table = "greptime_physical_table" +series_count = 16_384 +samples_per_series = 43_200 +sample_chunk_size = 1_440 +flush_every_sample_chunks = 1 +start_unix_millis = 1_704_067_200_000 # 2024-01-01T00:00:00Z +step_millis = 60_000 +chunk_series_count = 128 +timeout_seconds = 900 +visibility_timeout_seconds = 900 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 1_000_000 +max_concurrent_flushes = 256 +worker_channel_capacity = 65526 +max_inflight_requests = 3000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 20 +min_files_with_column = 20 + +[[scenario.queries]] +name = "selector_1d_remote_write_heavy" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704153600, '5m') prom_remote_write_heavy_storage{host=~'host.*'}" +warmup = 1 +iterations = 2 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 100 + +[[scenario.queries]] +name = "selector_7d_remote_write_heavy" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704672000, '1h') prom_remote_write_heavy_storage{host=~'host.*'}" +warmup = 1 +iterations = 2 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 100 + +[[scenario.queries]] +name = "selector_30d_remote_write_heavy" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1706659200, '1h') prom_remote_write_heavy_storage{host=~'host.*'}" +warmup = 1 +iterations = 2 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 100