mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-07-07 14:30:39 +00:00
chore: pick fixes and bump version to v1.1.2 (#8404)
* fix: improve Grafana metrics dashboards (#8298) * chore: initial changes Signed-off-by: evenyag <realevenyag@gmail.com> * feat: improve troubleshooting dashboard Signed-off-by: evenyag <realevenyag@gmail.com> * chore: rm troubleshooting-dashboard.md Signed-off-by: evenyag <realevenyag@gmail.com> * chore: optimize metrics dashboard Signed-off-by: evenyag <realevenyag@gmail.com> * docs: move troubleshooting-dashboard.md Signed-off-by: evenyag <realevenyag@gmail.com> * chore: move mito gc duration panel Signed-off-by: evenyag <realevenyag@gmail.com> * chore: cleanup the dashboard - Overview trend panels are now aggregate-only: - Total Ingestion Rate Trend - Total Query Rate Trend - Protocol breakdowns remain in Ingestion and Queries. - Mito Backpressure and Failures no longer duplicates scan/GC signals. - Removed Write Stall per Instance. - Split Object Store and WAL into collapsed Object Store and collapsed WAL. - Moved WAL/logstore panels out of Storage into WAL. - Normalized OpenDAL “other request” matchers. - Normalized trigger elapsed p99/p75/avg aggregation. - Regenerated standalone JSON and dashboard YAML/Markdown. - Updated docs/troubleshooting-dashboard.md. Signed-off-by: evenyag <realevenyag@gmail.com> * fix: rearrange metasrv dashboard panels Signed-off-by: evenyag <realevenyag@gmail.com> * feat: improve troubleshooting dashboard layout Signed-off-by: evenyag <realevenyag@gmail.com> * docs: remove obsolete troubleshooting dashboard doc Signed-off-by: evenyag <realevenyag@gmail.com> * fix: correct cluster dashboard panel queries (missing _bucket, raw counters, rate normalization) Signed-off-by: evenyag <realevenyag@gmail.com> * fix: correct trigger panel datasource, collapse flush/compaction, split request latency panels Signed-off-by: evenyag <realevenyag@gmail.com> * fix: update grafana metrics dashboard panels Signed-off-by: evenyag <realevenyag@gmail.com> * fix: correct Grafana dashboard units Signed-off-by: evenyag <realevenyag@gmail.com> * chore: regenerate Grafana dashboards Signed-off-by: evenyag <realevenyag@gmail.com> * fix: use throughput unit for index IO bytes Signed-off-by: evenyag <realevenyag@gmail.com> --------- Signed-off-by: evenyag <realevenyag@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: redact Kafka SASL password in debug output (#8337) ## Summary - Mask `KafkaClientSaslConfig` password fields in debug output while keeping usernames visible. - Cover metasrv WAL debug output with a regression test. ## Files - `src/common/wal/src/config/kafka/common.rs` - `src/common/wal/src/config.rs` Signed-off-by: Lei, HUANG <mrsatangel@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(query): run optimizer rules before MergeScan (#8339) * fix(query): push down join filters before MergeScan Signed-off-by: discord9 <discord9@163.com> * fix(query): run optimizer before MergeScan pushdown Signed-off-by: discord9 <discord9@163.com> * fix(query): narrow pre-MergeScan filter pushdown Signed-off-by: discord9 <discord9@163.com> * fix(query): refine pre-MergeScan optimizer prepass Signed-off-by: discord9 <discord9@163.com> * fix(query): satisfy predicate extractor clippy Signed-off-by: discord9 <discord9@163.com> * test(query): cover pre-MergeScan optimizer edges Signed-off-by: discord9 <discord9@163.com> * test(query): cover set comparison prepass Signed-off-by: discord9 <discord9@163.com> * fix(query): guard remote scan filter pushdown Signed-off-by: discord9 <discord9@163.com> * fix(query): preserve subquery planning errors Signed-off-by: discord9 <discord9@163.com> * fix(query): preserve usable scan predicates Signed-off-by: discord9 <discord9@163.com> * fix(query): simplify scan predicate extraction Signed-off-by: discord9 <discord9@163.com> * fix(query): keep scan filter extraction scoped Signed-off-by: discord9 <discord9@163.com> * docs(query): explain pre-MergeScan optimizer Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: preserve bulk write grpc error details (#8349) Signed-off-by: jeremyhi <fengjiachun@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: include index files in GC listing (#8327) * fix: include index files in GC listing Signed-off-by: discord9 <discord9@163.com> * chore: filter GC index listing to puffins Signed-off-by: discord9 <discord9@163.com> * chore: simplify GC index listing stream Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: stream tables for prometheus label discovery (#8341) Signed-off-by: Ritwij Aryan Parmar <ritwij.aryan.parmar@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: account parquet metadata cache size (#8368) * fix: account parquet metadata cache size Use Parquet metadata memory sizing for SST metadata cache weight and add regression coverage for byte-array page-index buffers. Signed-off-by: Lei, HUANG <mrsatangel@gmail.com> * fix: saturate sst meta cache weight Signed-off-by: Lei, HUANG <mrsatangel@gmail.com> --------- Signed-off-by: Lei, HUANG <mrsatangel@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: respect gc mailbox timeout for admin gc (#8363) Signed-off-by: discord9 <discord9@163.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: record catalog and schema in slow queries (#8387) * fix: record catalog and schema in slow queries Add catalog and schema context to slow query records while appending the new columns after existing fields to preserve column order. - `src/common/frontend/src/slow_query_event.rs`: extend `SlowQueryEvent` schema and rows with `catalog_name` and `schema_name`, and cover append-only ordering. - `src/catalog/src/process_manager.rs`: carry catalog and schema through `SlowQueryTimer`. - `src/frontend/src/instance.rs`: capture context for SQL, plan, and PromQL slow query timers. - `tests-integration/tests/sql.rs`: assert MySQL and PostgreSQL slow query records include catalog and schema. Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix: address slow query review comment Use `String::clone` when writing slow query catalog and schema values. Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix: keep slow query schema only Remove the slow query `catalog_name` column and keep `schema_name` as a non-null tag dimension. - `src/common/frontend/src/slow_query_event.rs`: expose only `schema_name` in `SlowQueryEvent` rows and mark it as a tag. - `src/catalog/src/process_manager.rs`: stop carrying catalog context in `SlowQueryTimer`. - `src/frontend/src/instance.rs`: pass only schema context to slow query timers. - `tests-integration/tests/sql.rs`: assert slow query records include `schema_name` without `catalog_name`. Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix: schema name semantic should be field Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix: typo Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> --------- Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: invalidate comment DDL cache and lock by object ID (#8390) * fix: invalidate comment ddl cache locally Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: fix typos Signed-off-by: WenyXu <wenymedia@gmail.com> * chore: apply suggestions Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com> * chore: client_ip error logs skip internal API (#8362) * chore: client_ip error logs skip internal API Signed-off-by: shuiyisong <xixing.sys@gmail.com> * fix: fmt Signed-off-by: shuiyisong <xixing.sys@gmail.com> * chore: use const Signed-off-by: shuiyisong <xixing.sys@gmail.com> * chore: use const Signed-off-by: shuiyisong <xixing.sys@gmail.com> --------- Signed-off-by: shuiyisong <xixing.sys@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * feat: update dashboard to v0.13.6 (#8369) Signed-off-by: WenyXu <wenymedia@gmail.com> * chore: use ENV for building dashboard (#8384) Signed-off-by: shuiyisong <xixing.sys@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: handle PromQL time binary aggregation (#8398) Signed-off-by: jeremyhi <fengjiachun@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * perf(mito): prune files by manifest time range (#8352) * perf(mito): prune files by manifest time range Signed-off-by: discord9 <discord9@163.com> * chore(mito): address file pruning review Signed-off-by: discord9 <discord9@163.com> * chore(mito): remove verbose file pruning log Signed-off-by: discord9 <discord9@163.com> * chore(mito): expose file pruning metric Signed-off-by: discord9 <discord9@163.com> * chore(mito): shorten file pruning metric Signed-off-by: discord9 <discord9@163.com> * test(mito): cover file pruning edge cases Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * perf(mito): skip manifest-pruned file ranges (#8366) * perf(mito): skip manifest-pruned file ranges Signed-off-by: discord9 <discord9@163.com> * test(mito): allow empty prune benchmark output Signed-off-by: discord9 <discord9@163.com> * fix(mito): avoid caching stale pruned builders Signed-off-by: discord9 <discord9@163.com> * chore(mito): address pruner clippy Signed-off-by: discord9 <discord9@163.com> * fix(mito): account worker pruner builder metrics Signed-off-by: discord9 <discord9@163.com> * test(mito): keep empty prune benchmark local Signed-off-by: discord9 <discord9@163.com> * refactor(mito): share manifest-pruned range skip Signed-off-by: discord9 <discord9@163.com> * chore(mito): shorten prune cache comment Signed-off-by: discord9 <discord9@163.com> * fix(mito): keep manifest prune state in pruner Signed-off-by: discord9 <discord9@163.com> * test(mito): cover manifest prune fast skip edge cases Signed-off-by: discord9 <discord9@163.com> * chore: fix typo in logical table alter Signed-off-by: discord9 <discord9@163.com> * chore(mito): address pruner review comments Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com> Signed-off-by: WenyXu <wenymedia@gmail.com> * chore: bump version to v1.1.2 Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: evenyag <realevenyag@gmail.com> Signed-off-by: WenyXu <wenymedia@gmail.com> Signed-off-by: Lei, HUANG <mrsatangel@gmail.com> Signed-off-by: discord9 <discord9@163.com> Signed-off-by: jeremyhi <fengjiachun@gmail.com> Signed-off-by: Ritwij Aryan Parmar <ritwij.aryan.parmar@gmail.com> Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> Signed-off-by: shuiyisong <xixing.sys@gmail.com> Co-authored-by: Yingwen <realevenyag@gmail.com> Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com> Co-authored-by: discord9 <discord9@163.com> Co-authored-by: jeremyhi <jiachun_feng@proton.me> Co-authored-by: Ritwij Aryan Parmar <88580521+RitwijParmar@users.noreply.github.com> Co-authored-by: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Co-authored-by: sun <sunchang_long@163.com>
This commit is contained in:
Weny Xu
GitHub
152
Cargo.lock
generated
152
Cargo.lock
generated
@@ -213,7 +213,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"
|
||||
|
||||
[[package]]
|
||||
name = "api"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"arrow-schema 58.3.0",
|
||||
"common-base",
|
||||
@@ -936,7 +936,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||
|
||||
[[package]]
|
||||
name = "auth"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -1574,7 +1574,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cache"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"catalog",
|
||||
"common-error",
|
||||
@@ -1610,7 +1610,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "catalog"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arrow 58.3.0",
|
||||
@@ -1945,7 +1945,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
|
||||
|
||||
[[package]]
|
||||
name = "cli"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
@@ -2004,7 +2004,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "client"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arc-swap",
|
||||
@@ -2036,7 +2036,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"snafu 0.8.6",
|
||||
"store-api",
|
||||
"substrait 1.1.1",
|
||||
"substrait 1.1.2",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic 0.14.2",
|
||||
@@ -2085,7 +2085,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cmd"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -2235,7 +2235,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-base"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"anymap2",
|
||||
@@ -2255,14 +2255,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-catalog"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"const_format",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "common-config"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-base",
|
||||
"common-error",
|
||||
@@ -2286,7 +2286,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-datasource"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"arrow 58.3.0",
|
||||
"arrow-schema 58.3.0",
|
||||
@@ -2322,7 +2322,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-decimal"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"bigdecimal 0.4.8",
|
||||
"common-error",
|
||||
@@ -2335,7 +2335,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-error"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-macro",
|
||||
"http 1.3.1",
|
||||
@@ -2346,7 +2346,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-event-recorder"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -2369,7 +2369,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-frontend"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -2390,7 +2390,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-function"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"api",
|
||||
@@ -2453,7 +2453,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-greptimedb-telemetry"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"common-runtime",
|
||||
@@ -2470,7 +2470,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-grpc"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arrow-flight",
|
||||
@@ -2505,7 +2505,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-grpc-expr"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"common-base",
|
||||
@@ -2525,7 +2525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-macro"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"greptime-proto",
|
||||
"once_cell",
|
||||
@@ -2536,7 +2536,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-mem-prof"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"common-error",
|
||||
@@ -2552,7 +2552,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-memory-manager"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-error",
|
||||
"common-macro",
|
||||
@@ -2564,7 +2564,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-meta"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"anymap2",
|
||||
"api",
|
||||
@@ -2635,7 +2635,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-options"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-grpc",
|
||||
"humantime-serde",
|
||||
@@ -2645,11 +2645,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-plugins"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
|
||||
[[package]]
|
||||
name = "common-pprof"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-error",
|
||||
"common-macro",
|
||||
@@ -2660,7 +2660,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-procedure"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-stream",
|
||||
@@ -2689,7 +2689,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-procedure-test"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"common-procedure",
|
||||
@@ -2699,7 +2699,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-query"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -2729,7 +2729,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-recordbatch"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"common-base",
|
||||
@@ -2754,7 +2754,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-runtime"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"clap",
|
||||
@@ -2783,7 +2783,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-session"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"strum 0.27.1",
|
||||
@@ -2791,7 +2791,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-sql"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"arrow-schema 58.3.0",
|
||||
"common-base",
|
||||
@@ -2811,7 +2811,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-stat"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-base",
|
||||
"common-runtime",
|
||||
@@ -2826,7 +2826,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-telemetry"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"common-base",
|
||||
@@ -2855,7 +2855,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-test-util"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"client",
|
||||
"common-grpc",
|
||||
@@ -2868,7 +2868,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-time"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"arrow 58.3.0",
|
||||
"chrono",
|
||||
@@ -2886,7 +2886,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-version"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"cargo-manifest",
|
||||
"const_format",
|
||||
@@ -2896,7 +2896,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-wal"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-base",
|
||||
"common-error",
|
||||
@@ -2919,7 +2919,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "common-workload"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"common-telemetry",
|
||||
"serde",
|
||||
@@ -4360,7 +4360,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datanode"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arrow-flight",
|
||||
@@ -4434,7 +4434,7 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
|
||||
|
||||
[[package]]
|
||||
name = "datatypes"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"arrow 58.3.0",
|
||||
"arrow-array 58.3.0",
|
||||
@@ -5171,7 +5171,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "file-engine"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -5302,7 +5302,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8"
|
||||
|
||||
[[package]]
|
||||
name = "flow"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arrow 58.3.0",
|
||||
@@ -5371,7 +5371,7 @@ dependencies = [
|
||||
"sql",
|
||||
"store-api",
|
||||
"strum 0.27.1",
|
||||
"substrait 1.1.1",
|
||||
"substrait 1.1.2",
|
||||
"table",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -5454,7 +5454,7 @@ checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
|
||||
|
||||
[[package]]
|
||||
name = "frontend"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arc-swap",
|
||||
@@ -6745,7 +6745,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "index"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"asynchronous-codec",
|
||||
@@ -7844,7 +7844,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "log-query"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"common-error",
|
||||
@@ -7856,7 +7856,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log-store"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
@@ -8175,7 +8175,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "meta-client"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -8207,7 +8207,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "meta-srv"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -8307,7 +8307,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metric-engine"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"aquamarine",
|
||||
@@ -8408,7 +8408,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mito-codec"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"bytes",
|
||||
@@ -8433,7 +8433,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mito2"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"aquamarine",
|
||||
@@ -9222,7 +9222,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "object-store"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -9777,7 +9777,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "operator"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"api",
|
||||
@@ -9836,7 +9836,7 @@ dependencies = [
|
||||
"sql",
|
||||
"sqlparser",
|
||||
"store-api",
|
||||
"substrait 1.1.1",
|
||||
"substrait 1.1.2",
|
||||
"table",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
@@ -10121,7 +10121,7 @@ checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699"
|
||||
|
||||
[[package]]
|
||||
name = "partition"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
@@ -10515,7 +10515,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pipeline"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"api",
|
||||
@@ -10672,7 +10672,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "plugins"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"auth",
|
||||
"catalog",
|
||||
@@ -11012,7 +11012,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "promql"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"async-trait",
|
||||
@@ -11364,7 +11364,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "puffin"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
@@ -11426,7 +11426,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "query"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"api",
|
||||
@@ -11496,7 +11496,7 @@ dependencies = [
|
||||
"sql",
|
||||
"sqlparser",
|
||||
"store-api",
|
||||
"substrait 1.1.1",
|
||||
"substrait 1.1.2",
|
||||
"table",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -13068,7 +13068,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "servers"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"api",
|
||||
@@ -13206,7 +13206,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "session"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"api",
|
||||
@@ -13559,7 +13559,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sql"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arrow-buffer 58.3.0",
|
||||
@@ -13620,7 +13620,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sqlness-runner"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"clap",
|
||||
@@ -13900,7 +13900,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "standalone"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"catalog",
|
||||
@@ -13944,7 +13944,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "store-api"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"aquamarine",
|
||||
@@ -14137,7 +14137,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "substrait"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -14259,7 +14259,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "table"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arc-swap",
|
||||
@@ -14542,7 +14542,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
|
||||
|
||||
[[package]]
|
||||
name = "tests-fuzz"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"async-trait",
|
||||
@@ -14587,7 +14587,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tests-integration"
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"api",
|
||||
"arrow-flight",
|
||||
@@ -14667,7 +14667,7 @@ dependencies = [
|
||||
"sqlx",
|
||||
"standalone",
|
||||
"store-api",
|
||||
"substrait 1.1.1",
|
||||
"substrait 1.1.2",
|
||||
"table",
|
||||
"tempfile",
|
||||
"time",
|
||||
|
||||
@@ -75,7 +75,7 @@ members = [
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "1.1.1"
|
||||
version = "1.1.2"
|
||||
edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,144 +4,154 @@
|
||||
| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` |
|
||||
| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- |
|
||||
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` |
|
||||
| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
|
||||
| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
|
||||
| Total Query Rate | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `stat` | Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. | `prometheus` | `reqps` | `queries` |
|
||||
| User-facing Error Rate | `sum(rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))` | `stat` | Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. | `prometheus` | `eps` | `errors` |
|
||||
| Recent Restarts | `sum(changes(process_start_time_seconds[$__range]))` | `stat` | Process restarts over the selected time range across GreptimeDB roles. | `prometheus` | `short` | `restarts` |
|
||||
| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`<br/>`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`<br/>`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`<br/>`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- |
|
||||
| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`<br/>`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`<br/>`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`<br/>`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- |
|
||||
| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
|
||||
| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
|
||||
| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`<br/>`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`<br/>`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- |
|
||||
| Total Ingestion Rate Trend | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. | `prometheus` | `rowsps` | `ingestion` |
|
||||
| Total Query Rate Trend | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. | `prometheus` | `reqps` | `queries` |
|
||||
| HTTP Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))`<br/>`sum(rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. | `prometheus` | `s` | `http-p99` |
|
||||
| gRPC Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`sum(rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Tail and average latency for gRPC requests served by frontends. | `prometheus` | `s` | `grpc-p99` |
|
||||
# Ingestion
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion rate.<br/><br/>Here we listed 3 primary protocols:<br/><br/>- Prometheus remote write<br/>- Greptime's gRPC API (when using our ingest SDK)<br/>- Log ingestion http API<br/> | `prometheus` | `rowsps` | `ingestion` |
|
||||
| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`<br/>`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.<br/><br/>Here we listed 3 primary protocols:<br/><br/>- Prometheus remote write<br/>- Greptime's gRPC API (when using our ingest SDK)<br/>- Log ingestion http API<br/> | `prometheus` | `rowsps` | `http-logs` |
|
||||
# Queries
|
||||
| Ingestion Rate by Protocol | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_prometheus_remote_write_samples{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_loki_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_elasticsearch_logs_docs_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_otlp_metrics_rows{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_otlp_logs_rows{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_otlp_traces_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Rows, samples, or documents ingested by primary observability and table-ingestion protocols. | `prometheus` | `rowsps` | `table-operator` |
|
||||
| Ingestion Latency by Protocol | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`sum(rate(greptime_servers_http_prometheus_write_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_otlp_logs_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_otlp_traces_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. | `prometheus` | `s` | `prometheus-write` |
|
||||
| Bulk Insert Message Rows and Size | `sum(rate(greptime_table_operator_bulk_insert_message_rows_sum{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_table_operator_bulk_insert_message_size_sum{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. | `prometheus` | `rowsps` | `rows` |
|
||||
| Prom Store Flush Pipeline | `sum(rate(greptime_prom_store_flush_total{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_prom_store_flush_rows_sum{instance=~"$frontend"}[$__rate_interval]))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Remote-write pending-row flush operations, flushed rows, and p99 flush latency. | `prometheus` | `short` | `flush-ops` |
|
||||
| OTLP Trace Failures | `sum(rate(greptime_frontend_otlp_traces_failure_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | OTLP trace ingestion failures reported by frontends. | `prometheus` | `eps` | `trace-failures` |
|
||||
# Health
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.<br/><br/>Here we listed 3 main protocols:<br/>- MySQL<br/>- Postgres<br/>- Prometheus API<br/><br/>Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` |
|
||||
| Protocol Error Rates | `sum by (protocol) (rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum by (code) (rate(greptime_servers_auth_failure_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics",code!~"2.."}[$__rate_interval]))`<br/>`sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend",code!~"0\|OK"}[$__rate_interval]))` | `timeseries` | User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. | `prometheus` | `eps` | `server-{{protocol}}` |
|
||||
| Frontend and Query Rejections | `sum(rate(greptime_servers_request_memory_rejected_total{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))` | `timeseries` | Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. | `prometheus` | `rps` | `request-memory` |
|
||||
| Datanode Write Failures | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. | `prometheus` | `eps` | `region-request-[{{instance}}]-[{{pod}}]` |
|
||||
| Buffered Ingestion Loss | `sum(rate(greptime_pending_rows_flush_failures{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_pending_rows_flush_dropped_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. | `prometheus` | `eps` | `flush-failures` |
|
||||
| Mito Backpressure and Failures | `sum(rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. | `prometheus` | `eps` | `write-reject` |
|
||||
| Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` |
|
||||
| OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode",error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` |
|
||||
| Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`<br/>`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` |
|
||||
| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` |
|
||||
| Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` |
|
||||
# Capacity
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Runtime Threads | `sum by (instance, pod) (greptime_runtime_threads_alive)`<br/>`sum by (instance, pod) (greptime_runtime_threads_idle)` | `timeseries` | Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. | `prometheus` | `short` | `alive-[{{instance}}]-[{{pod}}]` |
|
||||
| Request Memory Utilization | `sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes{instance=~"$frontend"}) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes{instance=~"$frontend"})` | `timeseries` | Frontend request memory usage divided by configured request memory limit. | `prometheus` | `percentunit` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Query Memory Usage | `sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)` | `timeseries` | Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Scan and Compaction Memory | `sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes{instance=~"$datanode"})`<br/>`sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes{instance=~"$datanode"})`<br/>`sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes{instance=~"$datanode"})` | `timeseries` | Datanode scan memory usage and compaction memory utilization. | `prometheus` | `bytes` | `scan-[{{instance}}]-[{{pod}}]` |
|
||||
| Write Buffer and Active Stalling | `sum by (instance, pod) (greptime_mito_write_buffer_bytes{instance=~"$datanode"})`<br/>`sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})` | `timeseries` | Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. | `prometheus` | `bytes` | `buffer-[{{instance}}]-[{{pod}}]` |
|
||||
| Prom Store Backlog | `sum by (instance, pod) (greptime_prom_store_pending_rows{instance=~"$frontend"})`<br/>`sum by (instance, pod) (greptime_prom_store_pending_batches{instance=~"$frontend"})`<br/>`sum by (instance, pod) (greptime_prom_store_pending_workers{instance=~"$frontend"})` | `timeseries` | Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. | `prometheus` | `short` | `rows-[{{instance}}]-[{{pod}}]` |
|
||||
| Inflight Flush and Compaction | `sum by (instance, pod) (greptime_mito_inflight_flush_count{instance=~"$datanode"})`<br/>`sum by (instance, pod) (greptime_mito_inflight_compaction_count{instance=~"$datanode"})` | `timeseries` | Current in-flight flush and compaction tasks on datanodes. | `prometheus` | `short` | `flush-[{{instance}}]-[{{pod}}]` |
|
||||
# Resources
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$datanode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
|
||||
| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{instance=~"$datanode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$frontend"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{instance=~"$frontend"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` |
|
||||
| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$metasrv"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
|
||||
| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{instance=~"$datanode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{instance=~"$metasrv"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$flownode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$frontend"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$datanode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
|
||||
| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$metasrv"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
|
||||
| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{instance=~"$flownode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{instance=~"$flownode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
# Queries
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Query Rate by Protocol | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Query API call rates by protocol, collected from frontends. | `prometheus` | `reqps` | `mysql` |
|
||||
| Query Latency by Protocol | `histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`sum(rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_promql_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | p95, p99, and average query latency by main frontend protocol. | `prometheus` | `s` | `mysql-p95` |
|
||||
| Query Stage Latency | `histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))` | `timeseries` | p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. | `prometheus` | `s` | `p95-{{stage}}` |
|
||||
| Merge Scan Fan-out and Errors | `sum by (instance, pod) (greptime_query_merge_scan_regions)`<br/>`sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))` | `timeseries` | Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. | `prometheus` | `short` | `regions-[{{instance}}]-[{{pod}}]` |
|
||||
| Pushdown Fallback Errors | `sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))` | `timeseries` | Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. | `prometheus` | `eps` | `pushdown-fallback-errors` |
|
||||
| PromQL Series Count | `sum by (instance, pod) (greptime_promql_series_count)` | `timeseries` | Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Connections and Prepared Statements | `sum by (instance, pod) (greptime_servers_mysql_connection_count{instance=~"$frontend"})`<br/>`sum by (instance, pod) (greptime_servers_postgres_connection_count{instance=~"$frontend"})`<br/>`sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. | `prometheus` | `short` | `mysql-connections-[{{instance}}]-[{{pod}}]` |
|
||||
# Frontend Requests
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` |
|
||||
| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| HTTP P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))`<br/>`sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` |
|
||||
| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| gRPC P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
|
||||
| MySQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL P99 and average per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
|
||||
| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| PostgreSQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
# Frontend to Datanode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
|
||||
| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
|
||||
| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`<br/>`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
|
||||
# Mito Engine
|
||||
| Region Call P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))`<br/>`sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
|
||||
| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`<br/>`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
|
||||
# Datanode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Region Request Failures and Failed Inserts | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Datanode region request failures and failed inserts by instance. | `prometheus` | `eps` | `request-fail-[{{instance}}]-[{{pod}}]` |
|
||||
| Write Rejects and Stalls | `sum by (instance, pod) (rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})` | `timeseries` | Mito write rejects, write stall events, and active write stalling by datanode. | `prometheus` | `short` | `reject-[{{instance}}]-[{{pod}}]` |
|
||||
| Flush and Compaction Failures | `sum by (instance, pod) (rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flush and compaction failure rates by datanode. | `prometheus` | `eps` | `flush-[{{instance}}]-[{{pod}}]` |
|
||||
| Mito GC Health | `sum(rate(greptime_mito_gc_runs_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_files_deleted_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. | `prometheus` | `short` | `runs` |
|
||||
| Mito GC Duration | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | P99 and average Mito garbage-collection duration by stage. | `prometheus` | `s` | `{{stage}}-p99` |
|
||||
# Storage
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Request Wait P99 and Avg per Worker | `histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{worker}}]-p95` |
|
||||
| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{instance=~"$datanode"}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
|
||||
| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
|
||||
| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
|
||||
| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
|
||||
| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` |
|
||||
| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
|
||||
| Read Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Read Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Write Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Write Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Cached Bytes per Instance | `greptime_mito_cache_bytes{instance=~"$datanode"}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
|
||||
| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`<br/>`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
|
||||
| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`<br/>`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
|
||||
| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`<br/>`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
|
||||
| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`<br/>`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Active series and field-builder counts per memtable by instance. | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
|
||||
| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`<br/>`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
|
||||
| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
# OpenDAL
|
||||
# Flush and Compaction
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
|
||||
# Remote WAL
|
||||
| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
|
||||
| Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
|
||||
| Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` |
|
||||
| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
|
||||
| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
|
||||
| Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`<br/>`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` |
|
||||
# Index
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
|
||||
| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
|
||||
| Index Apply Elapsed Time | `histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. | `prometheus` | `s` | `{{type}}-p95` |
|
||||
| Index Create Elapsed Time | `histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. | `prometheus` | `s` | `{{type}}-{{stage}}-p95` |
|
||||
| Index Create Rows and Bytes | `sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))` | `timeseries` | Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. | `prometheus` | `rowsps` | `{{type}}-rows` |
|
||||
| Index Memory Usage | `greptime_index_apply_memory_usage`<br/>`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` |
|
||||
| Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` |
|
||||
| Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` |
|
||||
| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
|
||||
# Metasrv
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Inactive and Lease-expired Regions | `sum(greptime_meta_inactive_regions)`<br/>`sum(greptime_lease_expired_region{instance=~"$datanode"})` | `timeseries` | Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. | `prometheus` | `short` | `inactive-regions` |
|
||||
| Heartbeat Health | `sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))`<br/>`sum(greptime_meta_heartbeat_connection_num)`<br/>`sum(rate(greptime_frontend_heartbeat_send_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_heartbeat_recv_count{instance=~"$frontend"}[$__rate_interval]))`<br/>`sum(rate(greptime_datanode_heartbeat_send_count{instance=~"$datanode"}[$__rate_interval]))`<br/>`sum(rate(greptime_datanode_heartbeat_recv_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. | `prometheus` | `short` | `meta-recv-rate` |
|
||||
| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`<br/>`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` |
|
||||
| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
|
||||
| Region migration error | `rate(greptime_meta_region_migration_error[$__rate_interval])` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
|
||||
| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` |
|
||||
| Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` |
|
||||
| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
|
||||
| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))`<br/>`sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
|
||||
| Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) (
|
||||
rate(greptime_meta_handler_execute_bucket[$__rate_interval])
|
||||
))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.<br/> | `prometheus` | `s` | `{{pod}} {{name}} p90` |
|
||||
| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.<br/> | `prometheus` | `bytes` | `{{pod}}` |
|
||||
| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` |
|
||||
| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
|
||||
| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
|
||||
| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
|
||||
| Reconciliation stats | `greptime_meta_reconciliation_stats` | `timeseries` | Reconciliation stats | `prometheus` | `s` | `{{pod}}-{{table_type}}-{{type}}` |
|
||||
| Reconciliation steps | `histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
|
||||
# Flownode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
|
||||
| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
|
||||
| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
|
||||
| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
|
||||
| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
|
||||
# Trigger
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
|
||||
| Trigger Eval Elapsed | `histogram_quantile(0.99,
|
||||
rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
|
||||
)`<br/>`histogram_quantile(0.75,
|
||||
rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
|
||||
)` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
|
||||
| Send Alert Elapsed | `histogram_quantile(0.99,
|
||||
rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
|
||||
)`<br/>`histogram_quantile(0.75,
|
||||
rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
|
||||
)` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99` |
|
||||
| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
|
||||
| Save Alert Elapsed | `histogram_quantile(0.99,
|
||||
rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
|
||||
)`<br/>`histogram_quantile(0.75,
|
||||
rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
|
||||
)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` |
|
||||
| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
|
||||
))`<br/>`sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.<br/> | `prometheus` | `s` | `{{pod}} {{name}} p90` |
|
||||
| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.<br/> | `prometheus` | `bytes` | `{{pod}}` |
|
||||
| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Rate of heartbeats received by metasrv from datanodes and frontends. | `prometheus` | `s` | `{{pod}}` |
|
||||
| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))`<br/>`sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average latency of metasrv key-value store operations by op and target. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
|
||||
| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Rate of metasrv key-value store operations by op. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
|
||||
| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))` | `timeseries` | p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
|
||||
| Reconciliation stats | `rate(greptime_meta_reconciliation_stats[$__rate_interval])` | `timeseries` | Reconciliation stats | `prometheus` | `ops` | `{{pod}}-{{table_type}}-{{type}}` |
|
||||
| Reconciliation steps | `histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
|
||||
# Hotspot
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
@@ -240,3 +250,45 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
|
||||
| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
|
||||
| Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
|
||||
| Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
|
||||
# Object Store
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read\|Reader::read"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| List P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Other Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
|
||||
# WAL
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
|
||||
| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
|
||||
| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
|
||||
| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
|
||||
# Flownode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
|
||||
| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
|
||||
| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
|
||||
| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
|
||||
| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
|
||||
# Trigger
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
|
||||
| Trigger Eval Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `p99` |
|
||||
| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
|
||||
| Send Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `p99` |
|
||||
| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
|
||||
| Save Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `p99` |
|
||||
| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -4,144 +4,154 @@
|
||||
| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` |
|
||||
| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- |
|
||||
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` |
|
||||
| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
|
||||
| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
|
||||
| Total Query Rate | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `stat` | Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. | `prometheus` | `reqps` | `queries` |
|
||||
| User-facing Error Rate | `sum(rate(greptime_servers_error[$__rate_interval]))` | `stat` | Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. | `prometheus` | `eps` | `errors` |
|
||||
| Recent Restarts | `sum(changes(process_start_time_seconds[$__range]))` | `stat` | Process restarts over the selected time range across GreptimeDB roles. | `prometheus` | `short` | `restarts` |
|
||||
| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`<br/>`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`<br/>`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`<br/>`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- |
|
||||
| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`<br/>`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`<br/>`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`<br/>`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- |
|
||||
| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
|
||||
| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
|
||||
| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`<br/>`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`<br/>`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- |
|
||||
| Total Ingestion Rate Trend | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `timeseries` | Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. | `prometheus` | `rowsps` | `ingestion` |
|
||||
| Total Query Rate Trend | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. | `prometheus` | `reqps` | `queries` |
|
||||
| HTTP Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))`<br/>`sum(rate(greptime_servers_http_requests_elapsed_sum{path!~"/health\|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. | `prometheus` | `s` | `http-p99` |
|
||||
| gRPC Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | Tail and average latency for gRPC requests served by frontends. | `prometheus` | `s` | `grpc-p99` |
|
||||
# Ingestion
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Total ingestion rate.<br/><br/>Here we listed 3 primary protocols:<br/><br/>- Prometheus remote write<br/>- Greptime's gRPC API (when using our ingest SDK)<br/>- Log ingestion http API<br/> | `prometheus` | `rowsps` | `ingestion` |
|
||||
| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`<br/>`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.<br/><br/>Here we listed 3 primary protocols:<br/><br/>- Prometheus remote write<br/>- Greptime's gRPC API (when using our ingest SDK)<br/>- Log ingestion http API<br/> | `prometheus` | `rowsps` | `http-logs` |
|
||||
# Queries
|
||||
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `timeseries` | Total ingestion rate.<br/><br/>Here we listed 3 primary protocols:<br/><br/>- Prometheus remote write<br/>- Greptime's gRPC API (when using our ingest SDK)<br/>- Log ingestion http API<br/> | `prometheus` | `rowsps` | `ingestion` |
|
||||
| Ingestion Rate by Protocol | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))`<br/>`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`<br/>`sum(rate(greptime_servers_loki_logs_ingestion_counter[$__rate_interval]))`<br/>`sum(rate(greptime_servers_elasticsearch_logs_docs_count[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_otlp_metrics_rows[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_otlp_logs_rows[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_otlp_traces_rows[$__rate_interval]))` | `timeseries` | Rows, samples, or documents ingested by primary observability and table-ingestion protocols. | `prometheus` | `rowsps` | `table-operator` |
|
||||
| Ingestion Latency by Protocol | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_servers_http_prometheus_write_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_otlp_logs_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_otlp_traces_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. | `prometheus` | `s` | `prometheus-write` |
|
||||
| Bulk Insert Message Rows and Size | `sum(rate(greptime_table_operator_bulk_insert_message_rows_sum[$__rate_interval]))`<br/>`sum(rate(greptime_table_operator_bulk_insert_message_size_sum[$__rate_interval]))` | `timeseries` | Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. | `prometheus` | `rowsps` | `rows` |
|
||||
| Prom Store Flush Pipeline | `sum(rate(greptime_prom_store_flush_total[$__rate_interval]))`<br/>`sum(rate(greptime_prom_store_flush_rows_sum[$__rate_interval]))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket[$__rate_interval])))` | `timeseries` | Remote-write pending-row flush operations, flushed rows, and p99 flush latency. | `prometheus` | `short` | `flush-ops` |
|
||||
| OTLP Trace Failures | `sum(rate(greptime_frontend_otlp_traces_failure_count[$__rate_interval]))` | `timeseries` | OTLP trace ingestion failures reported by frontends. | `prometheus` | `eps` | `trace-failures` |
|
||||
# Health
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))`<br/>`sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))`<br/>`sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.<br/><br/>Here we listed 3 main protocols:<br/>- MySQL<br/>- Postgres<br/>- Prometheus API<br/><br/>Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` |
|
||||
| Protocol Error Rates | `sum by (protocol) (rate(greptime_servers_error[$__rate_interval]))`<br/>`sum by (code) (rate(greptime_servers_auth_failure_count[$__rate_interval]))`<br/>`sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics",code!~"2.."}[$__rate_interval]))`<br/>`sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{code!~"0\|OK"}[$__rate_interval]))` | `timeseries` | User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. | `prometheus` | `eps` | `server-{{protocol}}` |
|
||||
| Frontend and Query Rejections | `sum(rate(greptime_servers_request_memory_rejected_total[$__rate_interval]))`<br/>`sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))` | `timeseries` | Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. | `prometheus` | `rps` | `request-memory` |
|
||||
| Datanode Write Failures | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))` | `timeseries` | Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. | `prometheus` | `eps` | `region-request-[{{instance}}]-[{{pod}}]` |
|
||||
| Buffered Ingestion Loss | `sum(rate(greptime_pending_rows_flush_failures[$__rate_interval]))`<br/>`sum(rate(greptime_pending_rows_flush_dropped_rows[$__rate_interval]))` | `timeseries` | Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. | `prometheus` | `eps` | `flush-failures` |
|
||||
| Mito Backpressure and Failures | `sum(rate(greptime_mito_write_reject_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_write_stall_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_flush_failure_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_compaction_failure_total[$__rate_interval]))` | `timeseries` | Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. | `prometheus` | `eps` | `write-reject` |
|
||||
| Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` |
|
||||
| OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` |
|
||||
| Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`<br/>`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` |
|
||||
| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`<br/>`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` |
|
||||
| Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` |
|
||||
# Capacity
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Runtime Threads | `sum by (instance, pod) (greptime_runtime_threads_alive)`<br/>`sum by (instance, pod) (greptime_runtime_threads_idle)` | `timeseries` | Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. | `prometheus` | `short` | `alive-[{{instance}}]-[{{pod}}]` |
|
||||
| Request Memory Utilization | `sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes)` | `timeseries` | Frontend request memory usage divided by configured request memory limit. | `prometheus` | `percentunit` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Query Memory Usage | `sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)` | `timeseries` | Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Scan and Compaction Memory | `sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes)`<br/>`sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes)`<br/>`sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes)` | `timeseries` | Datanode scan memory usage and compaction memory utilization. | `prometheus` | `bytes` | `scan-[{{instance}}]-[{{pod}}]` |
|
||||
| Write Buffer and Active Stalling | `sum by (instance, pod) (greptime_mito_write_buffer_bytes)`<br/>`sum by (instance, pod) (greptime_mito_write_stalling_count)` | `timeseries` | Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. | `prometheus` | `bytes` | `buffer-[{{instance}}]-[{{pod}}]` |
|
||||
| Prom Store Backlog | `sum by (instance, pod) (greptime_prom_store_pending_rows)`<br/>`sum by (instance, pod) (greptime_prom_store_pending_batches)`<br/>`sum by (instance, pod) (greptime_prom_store_pending_workers)` | `timeseries` | Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. | `prometheus` | `short` | `rows-[{{instance}}]-[{{pod}}]` |
|
||||
| Inflight Flush and Compaction | `sum by (instance, pod) (greptime_mito_inflight_flush_count)`<br/>`sum by (instance, pod) (greptime_mito_inflight_compaction_count)` | `timeseries` | Current in-flight flush and compaction tasks on datanodes. | `prometheus` | `short` | `flush-[{{instance}}]-[{{pod}}]` |
|
||||
# Resources
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Datanode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
|
||||
| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Frontend Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` |
|
||||
| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
|
||||
| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Flownode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` |
|
||||
| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Frontend Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Datanode Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
|
||||
| Metasrv Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
|
||||
| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`<br/>`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
| Flownode Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`<br/>`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
|
||||
# Queries
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Query Rate by Protocol | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | Query API call rates by protocol, collected from frontends. | `prometheus` | `reqps` | `mysql` |
|
||||
| Query Latency by Protocol | `histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_servers_http_promql_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | p95, p99, and average query latency by main frontend protocol. | `prometheus` | `s` | `mysql-p95` |
|
||||
| Query Stage Latency | `histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))` | `timeseries` | p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. | `prometheus` | `s` | `p95-{{stage}}` |
|
||||
| Merge Scan Fan-out and Errors | `sum by (instance, pod) (greptime_query_merge_scan_regions)`<br/>`sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))` | `timeseries` | Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. | `prometheus` | `short` | `regions-[{{instance}}]-[{{pod}}]` |
|
||||
| Pushdown Fallback Errors | `sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))` | `timeseries` | Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. | `prometheus` | `eps` | `pushdown-fallback-errors` |
|
||||
| PromQL Series Count | `sum by (instance, pod) (greptime_promql_series_count)` | `timeseries` | Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Connections and Prepared Statements | `sum by (instance, pod) (greptime_servers_mysql_connection_count)`<br/>`sum by (instance, pod) (greptime_servers_postgres_connection_count)`<br/>`sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count[$__rate_interval]))` | `timeseries` | MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. | `prometheus` | `short` | `mysql-connections-[{{instance}}]-[{{pod}}]` |
|
||||
# Frontend Requests
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` |
|
||||
| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` |
|
||||
| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
|
||||
| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| HTTP P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))`<br/>`sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{path!~"/health\|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` |
|
||||
| gRPC P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))`<br/>`sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | gRPC P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
|
||||
| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| MySQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`<br/>`sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))` | `timeseries` | MySQL P99 and average per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
|
||||
| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| PostgreSQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`<br/>`sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))` | `timeseries` | PostgreSQL P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
# Frontend to Datanode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
|
||||
| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
|
||||
| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`<br/>`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
|
||||
# Mito Engine
|
||||
| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
|
||||
| Region Call P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket[$__rate_interval])))`<br/>`sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))` | `timeseries` | Region Call P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
|
||||
| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`<br/>`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
|
||||
# Datanode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
|
||||
| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
|
||||
| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
|
||||
| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
|
||||
| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` |
|
||||
| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
|
||||
| Cached Bytes per Instance | `greptime_mito_cache_bytes{}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
|
||||
| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`<br/>`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
|
||||
| Region Request Failures and Failed Inserts | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))` | `timeseries` | Datanode region request failures and failed inserts by instance. | `prometheus` | `eps` | `request-fail-[{{instance}}]-[{{pod}}]` |
|
||||
| Write Rejects and Stalls | `sum by (instance, pod) (rate(greptime_mito_write_reject_total[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_mito_write_stall_total[$__rate_interval]))`<br/>`sum by (instance, pod) (greptime_mito_write_stalling_count)` | `timeseries` | Mito write rejects, write stall events, and active write stalling by datanode. | `prometheus` | `short` | `reject-[{{instance}}]-[{{pod}}]` |
|
||||
| Flush and Compaction Failures | `sum by (instance, pod) (rate(greptime_mito_flush_failure_total[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_mito_compaction_failure_total[$__rate_interval]))` | `timeseries` | Mito flush and compaction failure rates by datanode. | `prometheus` | `eps` | `flush-[{{instance}}]-[{{pod}}]` |
|
||||
| Mito GC Health | `sum(rate(greptime_mito_gc_runs_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_files_deleted_total[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))`<br/>`sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))` | `timeseries` | Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. | `prometheus` | `short` | `runs` |
|
||||
| Mito GC Duration | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket[$__rate_interval])))`<br/>`sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count[$__rate_interval]))` | `timeseries` | P99 and average Mito garbage-collection duration by stage. | `prometheus` | `s` | `{{stage}}-p99` |
|
||||
# Storage
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))`<br/>`sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))` | `timeseries` | Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Request Wait P99 and Avg per Worker | `histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))`<br/>`sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count[$__rate_interval]))` | `timeseries` | Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{worker}}]-p95` |
|
||||
| Write Buffer per Instance | `greptime_mito_write_buffer_bytes` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Read Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Read Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Write Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Write Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
|
||||
| Cached Bytes per Instance | `greptime_mito_cache_bytes` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`<br/>`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
|
||||
| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`<br/>`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
|
||||
| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`<br/>`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Active series and field-builder counts per memtable by instance. | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
|
||||
| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`<br/>`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
|
||||
| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
# OpenDAL
|
||||
| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
|
||||
# Flush and Compaction
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
|
||||
# Remote WAL
|
||||
| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
|
||||
| Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`<br/>`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
|
||||
| Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval]))`<br/>`sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` |
|
||||
| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
|
||||
| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
|
||||
| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))`<br/>`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
|
||||
| Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`<br/>`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` |
|
||||
# Index
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
|
||||
| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
|
||||
| Index Apply Elapsed Time | `histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. | `prometheus` | `s` | `{{type}}-p95` |
|
||||
| Index Create Elapsed Time | `histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. | `prometheus` | `s` | `{{type}}-{{stage}}-p95` |
|
||||
| Index Create Rows and Bytes | `sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))` | `timeseries` | Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. | `prometheus` | `rowsps` | `{{type}}-rows` |
|
||||
| Index Memory Usage | `greptime_index_apply_memory_usage`<br/>`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` |
|
||||
| Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` |
|
||||
| Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` |
|
||||
| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`<br/>`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
|
||||
# Metasrv
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Inactive and Lease-expired Regions | `sum(greptime_meta_inactive_regions)`<br/>`sum(greptime_lease_expired_region)` | `timeseries` | Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. | `prometheus` | `short` | `inactive-regions` |
|
||||
| Heartbeat Health | `sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))`<br/>`sum(greptime_meta_heartbeat_connection_num)`<br/>`sum(rate(greptime_frontend_heartbeat_send_count[$__rate_interval]))`<br/>`sum(rate(greptime_frontend_heartbeat_recv_count[$__rate_interval]))`<br/>`sum(rate(greptime_datanode_heartbeat_send_count[$__rate_interval]))`<br/>`sum(rate(greptime_datanode_heartbeat_recv_count[$__rate_interval]))` | `timeseries` | Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. | `prometheus` | `short` | `meta-recv-rate` |
|
||||
| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`<br/>`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` |
|
||||
| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
|
||||
| Region migration error | `rate(greptime_meta_region_migration_error[$__rate_interval])` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
|
||||
| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` |
|
||||
| Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` |
|
||||
| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
|
||||
| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))`<br/>`sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
|
||||
| Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) (
|
||||
rate(greptime_meta_handler_execute_bucket[$__rate_interval])
|
||||
))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.<br/> | `prometheus` | `s` | `{{pod}} {{name}} p90` |
|
||||
| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.<br/> | `prometheus` | `bytes` | `{{pod}}` |
|
||||
| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` |
|
||||
| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
|
||||
| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
|
||||
| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
|
||||
| Reconciliation stats | `greptime_meta_reconciliation_stats` | `timeseries` | Reconciliation stats | `prometheus` | `s` | `{{pod}}-{{table_type}}-{{type}}` |
|
||||
| Reconciliation steps | `histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
|
||||
# Flownode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
|
||||
| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
|
||||
| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
|
||||
| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
|
||||
| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
|
||||
# Trigger
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
|
||||
| Trigger Eval Elapsed | `histogram_quantile(0.99,
|
||||
rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
|
||||
)`<br/>`histogram_quantile(0.75,
|
||||
rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
|
||||
)` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
|
||||
| Send Alert Elapsed | `histogram_quantile(0.99,
|
||||
rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
|
||||
)`<br/>`histogram_quantile(0.75,
|
||||
rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
|
||||
)` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99` |
|
||||
| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
|
||||
| Save Alert Elapsed | `histogram_quantile(0.99,
|
||||
rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
|
||||
)`<br/>`histogram_quantile(0.75,
|
||||
rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
|
||||
)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` |
|
||||
| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
|
||||
))`<br/>`sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.<br/> | `prometheus` | `s` | `{{pod}} {{name}} p90` |
|
||||
| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.<br/> | `prometheus` | `bytes` | `{{pod}}` |
|
||||
| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Rate of heartbeats received by metasrv from datanodes and frontends. | `prometheus` | `s` | `{{pod}}` |
|
||||
| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))`<br/>`sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average latency of metasrv key-value store operations by op and target. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
|
||||
| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Rate of metasrv key-value store operations by op. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
|
||||
| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))` | `timeseries` | p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
|
||||
| Reconciliation stats | `rate(greptime_meta_reconciliation_stats[$__rate_interval])` | `timeseries` | Reconciliation stats | `prometheus` | `ops` | `{{pod}}-{{table_type}}-{{type}}` |
|
||||
| Reconciliation steps | `histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
|
||||
# Hotspot
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
@@ -240,3 +250,45 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
|
||||
| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
|
||||
| Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
|
||||
| Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
|
||||
# Object Store
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Read P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read\|Reader::read"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"read\|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Write P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| List P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="list"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))` | `timeseries` | List P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
|
||||
| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Other Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])))`<br/>`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
|
||||
| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
|
||||
# WAL
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
|
||||
| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
|
||||
| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
|
||||
| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
|
||||
| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
|
||||
| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
|
||||
# Flownode
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
|
||||
| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`<br/>`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
|
||||
| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`<br/>`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
|
||||
| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
|
||||
| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
|
||||
# Trigger
|
||||
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Trigger Count | `greptime_trigger_count` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
|
||||
| Trigger Eval Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `p99` |
|
||||
| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
|
||||
| Send Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `p99` |
|
||||
| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
|
||||
| Save Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`<br/>`sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `p99` |
|
||||
| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,7 +6,11 @@ DAC_IMAGE=ghcr.io/zyy17/dac:20250423-522bd35
|
||||
|
||||
remove_instance_filters() {
|
||||
# Remove the instance filters for the standalone dashboards.
|
||||
sed -E 's/instance=~\\"(\$datanode|\$frontend|\$metasrv|\$flownode)\\",?//g' "$CLUSTER_DASHBOARD_DIR/dashboard.json" > "$STANDALONE_DASHBOARD_DIR/dashboard.json"
|
||||
sed -E 's/instance=~\\"(\$datanode|\$frontend|\$metasrv|\$flownode)\\"[[:space:]]*,?[[:space:]]*//g' "$CLUSTER_DASHBOARD_DIR/dashboard.json" \
|
||||
| sed -E 's/\{[[:space:]]*,[[:space:]]*/{/g' \
|
||||
| sed -E 's/,[[:space:]]*\}/}/g' \
|
||||
| sed -E 's/([A-Za-z_:][A-Za-z0-9_:]*)\{[[:space:]]*\}/\1/g' \
|
||||
> "$STANDALONE_DASHBOARD_DIR/dashboard.json"
|
||||
}
|
||||
|
||||
generate_intermediate_dashboards_and_docs() {
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
|
||||
set -ex
|
||||
set -e
|
||||
|
||||
declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
|
||||
declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
|
||||
declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
|
||||
OUT_DIR="${1:-$SCRIPT_DIR}"
|
||||
|
||||
RELEASE_VERSION="$(cat $STATIC_DIR/VERSION | tr -d '\t\r\n ')"
|
||||
DASHBOARD_REPOSITORY="${DASHBOARD_REPOSITORY:-GreptimeTeam/dashboard}"
|
||||
DASHBOARD_ASSET="${DASHBOARD_ASSET:-build.tar.gz}"
|
||||
DASHBOARD_GITHUB_TOKEN="${DASHBOARD_GITHUB_TOKEN:-${GH_TOKEN:-${GITHUB_TOKEN:-}}}"
|
||||
RELEASE_VERSION="${DASHBOARD_RELEASE_VERSION:-$(cat "$STATIC_DIR/VERSION" | tr -d '\t\r\n ')}"
|
||||
|
||||
echo "Downloading assets to dir: $OUT_DIR"
|
||||
cd $OUT_DIR
|
||||
@@ -22,8 +25,16 @@ fi
|
||||
function retry_fetch() {
|
||||
local url=$1
|
||||
local filename=$2
|
||||
local auth_args=()
|
||||
|
||||
curl --connect-timeout 10 --retry 3 -fsSL $url --output $filename || {
|
||||
if [[ -n "$DASHBOARD_GITHUB_TOKEN" ]]; then
|
||||
auth_args=(
|
||||
-H "Authorization: Bearer ${DASHBOARD_GITHUB_TOKEN}"
|
||||
-H "Accept: application/octet-stream"
|
||||
)
|
||||
fi
|
||||
|
||||
curl --connect-timeout 10 --retry 3 -fsSL "${auth_args[@]}" "$url" --output "$filename" || {
|
||||
echo "Failed to download $url"
|
||||
echo "You may try to set http_proxy and https_proxy environment variables."
|
||||
if [[ -z "$GITHUB_PROXY_URL" ]]; then
|
||||
@@ -36,10 +47,10 @@ function retry_fetch() {
|
||||
# Download the SHA256 checksum attached to the release. To verify the integrity
|
||||
# of the download, this checksum will be used to check the download tar file
|
||||
# containing the built dashboard assets.
|
||||
retry_fetch "${GITHUB_URL}/GreptimeTeam/dashboard/releases/download/${RELEASE_VERSION}/sha256.txt" sha256.txt
|
||||
retry_fetch "${GITHUB_URL}/${DASHBOARD_REPOSITORY}/releases/download/${RELEASE_VERSION}/sha256.txt" sha256.txt
|
||||
|
||||
# Download the tar file containing the built dashboard assets.
|
||||
retry_fetch "${GITHUB_URL}/GreptimeTeam/dashboard/releases/download/${RELEASE_VERSION}/build.tar.gz" build.tar.gz
|
||||
retry_fetch "${GITHUB_URL}/${DASHBOARD_REPOSITORY}/releases/download/${RELEASE_VERSION}/${DASHBOARD_ASSET}" "$DASHBOARD_ASSET"
|
||||
|
||||
# Verify the checksums match; exit if they don't.
|
||||
case "$(uname -s)" in
|
||||
@@ -55,8 +66,8 @@ case "$(uname -s)" in
|
||||
esac
|
||||
|
||||
# Extract the assets and clean up.
|
||||
tar -xzf build.tar.gz -C "$STATIC_DIR"
|
||||
tar -xzf "$DASHBOARD_ASSET" -C "$STATIC_DIR"
|
||||
rm sha256.txt
|
||||
rm build.tar.gz
|
||||
rm "$DASHBOARD_ASSET"
|
||||
|
||||
echo "Successfully download dashboard assets to $STATIC_DIR"
|
||||
|
||||
@@ -307,6 +307,7 @@ impl Debug for CancellableProcess {
|
||||
pub struct SlowQueryTimer {
|
||||
start: Instant,
|
||||
stmt: QueryStatement,
|
||||
schema_name: String,
|
||||
threshold: Duration,
|
||||
sample_ratio: f64,
|
||||
record_type: SlowQueriesRecordType,
|
||||
@@ -316,6 +317,7 @@ pub struct SlowQueryTimer {
|
||||
impl SlowQueryTimer {
|
||||
pub fn new(
|
||||
stmt: QueryStatement,
|
||||
schema_name: String,
|
||||
threshold: Duration,
|
||||
sample_ratio: f64,
|
||||
record_type: SlowQueriesRecordType,
|
||||
@@ -324,6 +326,7 @@ impl SlowQueryTimer {
|
||||
Self {
|
||||
start: Instant::now(),
|
||||
stmt,
|
||||
schema_name,
|
||||
threshold,
|
||||
sample_ratio,
|
||||
record_type,
|
||||
@@ -338,6 +341,7 @@ impl SlowQueryTimer {
|
||||
cost: elapsed.as_millis() as u64,
|
||||
threshold: self.threshold.as_millis() as u64,
|
||||
query: "".to_string(),
|
||||
schema_name: self.schema_name.clone(),
|
||||
|
||||
// The following fields are only used for PromQL queries.
|
||||
is_promql: false,
|
||||
@@ -388,6 +392,7 @@ impl SlowQueryTimer {
|
||||
cost = slow_query_event.cost,
|
||||
threshold = slow_query_event.threshold,
|
||||
query = slow_query_event.query,
|
||||
schema_name = slow_query_event.schema_name,
|
||||
is_promql = slow_query_event.is_promql,
|
||||
promql_range = slow_query_event.promql_range,
|
||||
promql_step = slow_query_event.promql_step,
|
||||
|
||||
@@ -24,6 +24,7 @@ pub const SLOW_QUERY_TABLE_NAME: &str = "slow_queries";
|
||||
pub const SLOW_QUERY_TABLE_COST_COLUMN_NAME: &str = "cost";
|
||||
pub const SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME: &str = "threshold";
|
||||
pub const SLOW_QUERY_TABLE_QUERY_COLUMN_NAME: &str = "query";
|
||||
pub const SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME: &str = "schema_name";
|
||||
pub const SLOW_QUERY_TABLE_TIMESTAMP_COLUMN_NAME: &str = "timestamp";
|
||||
pub const SLOW_QUERY_TABLE_IS_PROMQL_COLUMN_NAME: &str = "is_promql";
|
||||
pub const SLOW_QUERY_TABLE_PROMQL_START_COLUMN_NAME: &str = "promql_start";
|
||||
@@ -38,6 +39,7 @@ pub struct SlowQueryEvent {
|
||||
pub cost: u64,
|
||||
pub threshold: u64,
|
||||
pub query: String,
|
||||
pub schema_name: String,
|
||||
pub is_promql: bool,
|
||||
pub promql_range: Option<u64>,
|
||||
pub promql_step: Option<u64>,
|
||||
@@ -104,6 +106,12 @@ impl Event for SlowQueryEvent {
|
||||
semantic_type: SemanticType::Field.into(),
|
||||
..Default::default()
|
||||
},
|
||||
ColumnSchema {
|
||||
column_name: SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME.to_string(),
|
||||
datatype: ColumnDataType::String.into(),
|
||||
semantic_type: SemanticType::Field.into(),
|
||||
..Default::default()
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
@@ -118,6 +126,7 @@ impl Event for SlowQueryEvent {
|
||||
ValueData::U64Value(self.promql_step.unwrap_or(0)).into(),
|
||||
ValueData::TimestampMillisecondValue(self.promql_start.unwrap_or(0)).into(),
|
||||
ValueData::TimestampMillisecondValue(self.promql_end.unwrap_or(0)).into(),
|
||||
ValueData::StringValue(self.schema_name.clone()).into(),
|
||||
],
|
||||
}])
|
||||
}
|
||||
@@ -126,3 +135,54 @@ impl Event for SlowQueryEvent {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use api::v1::value::ValueData;
|
||||
use common_event_recorder::Event;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn slow_query_event_includes_schema() {
|
||||
let event = SlowQueryEvent {
|
||||
cost: 100,
|
||||
threshold: 10,
|
||||
query: "SELECT * FROM numbers".to_string(),
|
||||
schema_name: "public".to_string(),
|
||||
is_promql: false,
|
||||
promql_range: None,
|
||||
promql_step: None,
|
||||
promql_start: None,
|
||||
promql_end: None,
|
||||
};
|
||||
|
||||
let schema = event.extra_schema();
|
||||
let column_names = schema
|
||||
.iter()
|
||||
.map(|column| column.column_name.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
column_names,
|
||||
vec![
|
||||
SLOW_QUERY_TABLE_COST_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_IS_PROMQL_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_PROMQL_RANGE_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_PROMQL_STEP_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_PROMQL_START_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_PROMQL_END_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME,
|
||||
]
|
||||
);
|
||||
assert_eq!(schema[8].semantic_type, SemanticType::Field as i32);
|
||||
|
||||
let rows = event.extra_rows().unwrap();
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(
|
||||
rows[0].values[8].value_data,
|
||||
Some(ValueData::StringValue("public".to_string()))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use common_error::ext::BoxedError;
|
||||
use common_macro::admin_fn;
|
||||
use common_meta::rpc::procedure::{GcRegionsRequest, GcTableRequest};
|
||||
@@ -30,7 +28,6 @@ use snafu::{ResultExt, ensure};
|
||||
use crate::handlers::ProcedureServiceHandlerRef;
|
||||
use crate::helper::cast_u64;
|
||||
|
||||
const DEFAULT_GC_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const DEFAULT_FULL_FILE_LISTING: bool = false;
|
||||
|
||||
#[admin_fn(
|
||||
@@ -50,7 +47,7 @@ pub(crate) async fn gc_regions(
|
||||
.gc_regions(GcRegionsRequest {
|
||||
region_ids,
|
||||
full_file_listing,
|
||||
timeout: DEFAULT_GC_TIMEOUT,
|
||||
timeout: None,
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -77,7 +74,7 @@ pub(crate) async fn gc_table(
|
||||
schema_name,
|
||||
table_name,
|
||||
full_file_listing,
|
||||
timeout: DEFAULT_GC_TIMEOUT,
|
||||
timeout: None,
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -180,9 +177,18 @@ fn gc_table_signature() -> Signature {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use api::v1::meta::ReconcileRequest;
|
||||
use async_trait::async_trait;
|
||||
use catalog::CatalogManagerRef;
|
||||
use common_meta::rpc::procedure::{
|
||||
GcResponse, ManageRegionFollowerRequest, MigrateRegionRequest, ProcedureStateResponse,
|
||||
};
|
||||
use session::context::QueryContext;
|
||||
|
||||
use super::*;
|
||||
use crate::handlers::ProcedureServiceHandler;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gc_regions_params_with_full_file_listing() {
|
||||
@@ -217,4 +223,80 @@ mod tests {
|
||||
assert_eq!(table, "t");
|
||||
assert!(full_file_listing);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_regions_uses_meta_gc_timeout_config() {
|
||||
let handler = Arc::new(MockProcedureServiceHandler::default());
|
||||
let handler_ref: ProcedureServiceHandlerRef = handler.clone();
|
||||
let params = vec![ValueRef::UInt64(1), ValueRef::Boolean(true)];
|
||||
|
||||
super::gc_regions(&handler_ref, &QueryContext::arc(), ¶ms)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let request = handler.gc_regions_request.lock().unwrap().clone().unwrap();
|
||||
assert_eq!(request.region_ids, vec![1]);
|
||||
assert!(request.full_file_listing);
|
||||
assert_eq!(request.timeout, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_table_uses_meta_gc_timeout_config() {
|
||||
let handler = Arc::new(MockProcedureServiceHandler::default());
|
||||
let handler_ref: ProcedureServiceHandlerRef = handler.clone();
|
||||
let params = vec![ValueRef::String("public.t"), ValueRef::Boolean(true)];
|
||||
|
||||
super::gc_table(&handler_ref, &QueryContext::arc(), ¶ms)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let request = handler.gc_table_request.lock().unwrap().clone().unwrap();
|
||||
assert_eq!(request.catalog_name, "greptime");
|
||||
assert_eq!(request.schema_name, "public");
|
||||
assert_eq!(request.table_name, "t");
|
||||
assert!(request.full_file_listing);
|
||||
assert_eq!(request.timeout, None);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockProcedureServiceHandler {
|
||||
gc_regions_request: Mutex<Option<GcRegionsRequest>>,
|
||||
gc_table_request: Mutex<Option<GcTableRequest>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProcedureServiceHandler for MockProcedureServiceHandler {
|
||||
async fn migrate_region(&self, _request: MigrateRegionRequest) -> Result<Option<String>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
async fn reconcile(&self, _request: ReconcileRequest) -> Result<Option<String>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
async fn query_procedure_state(&self, _pid: &str) -> Result<ProcedureStateResponse> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
async fn manage_region_follower(
|
||||
&self,
|
||||
_request: ManageRegionFollowerRequest,
|
||||
) -> Result<()> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn catalog_manager(&self) -> &CatalogManagerRef {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
async fn gc_regions(&self, request: GcRegionsRequest) -> Result<GcResponse> {
|
||||
*self.gc_regions_request.lock().unwrap() = Some(request);
|
||||
Ok(GcResponse::default())
|
||||
}
|
||||
|
||||
async fn gc_table(&self, request: GcTableRequest) -> Result<GcResponse> {
|
||||
*self.gc_table_request.lock().unwrap() = Some(request);
|
||||
Ok(GcResponse::default())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use common_catalog::format_full_table_name;
|
||||
use common_catalog::{format_full_flow_name, format_full_table_name};
|
||||
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
|
||||
use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status};
|
||||
use common_telemetry::tracing::info;
|
||||
@@ -38,8 +38,8 @@ use crate::key::flow::flow_info::{FlowInfoKey, FlowInfoValue};
|
||||
use crate::key::table_info::{TableInfoKey, TableInfoValue};
|
||||
use crate::key::table_name::TableNameKey;
|
||||
use crate::key::{DeserializedValueWithBytes, FlowId, MetadataKey, MetadataValue};
|
||||
use crate::lock_key::{CatalogLock, FlowNameLock, SchemaLock, TableNameLock};
|
||||
use crate::rpc::ddl::{CommentObjectType, CommentOnTask};
|
||||
use crate::lock_key::{CatalogLock, FlowLock, FlowNameLock, SchemaLock, TableLock, TableNameLock};
|
||||
use crate::rpc::ddl::{CommentObjectId, CommentObjectType, CommentOnTask};
|
||||
use crate::rpc::store::PutRequest;
|
||||
|
||||
pub struct CommentOnProcedure {
|
||||
@@ -106,26 +106,29 @@ impl CommentOnProcedure {
|
||||
}
|
||||
|
||||
async fn prepare_table_or_column(&mut self) -> Result<()> {
|
||||
let table_name_key = TableNameKey::new(
|
||||
&self.data.catalog_name,
|
||||
&self.data.schema_name,
|
||||
&self.data.object_name,
|
||||
);
|
||||
let table_id = if let Some(table_id) = self.data.table_id {
|
||||
table_id
|
||||
} else {
|
||||
let table_name_key = TableNameKey::new(
|
||||
&self.data.catalog_name,
|
||||
&self.data.schema_name,
|
||||
&self.data.object_name,
|
||||
);
|
||||
|
||||
let table_id = self
|
||||
.context
|
||||
.table_metadata_manager
|
||||
.table_name_manager()
|
||||
.get(table_name_key)
|
||||
.await?
|
||||
.with_context(|| TableNotFoundSnafu {
|
||||
table_name: format_full_table_name(
|
||||
&self.data.catalog_name,
|
||||
&self.data.schema_name,
|
||||
&self.data.object_name,
|
||||
),
|
||||
})?
|
||||
.table_id();
|
||||
self.context
|
||||
.table_metadata_manager
|
||||
.table_name_manager()
|
||||
.get(table_name_key)
|
||||
.await?
|
||||
.with_context(|| TableNotFoundSnafu {
|
||||
table_name: format_full_table_name(
|
||||
&self.data.catalog_name,
|
||||
&self.data.schema_name,
|
||||
&self.data.object_name,
|
||||
),
|
||||
})?
|
||||
.table_id()
|
||||
};
|
||||
|
||||
let table_info = self
|
||||
.context
|
||||
@@ -198,17 +201,22 @@ impl CommentOnProcedure {
|
||||
}
|
||||
|
||||
async fn prepare_flow(&mut self) -> Result<()> {
|
||||
let flow_name_value = self
|
||||
.context
|
||||
.flow_metadata_manager
|
||||
.flow_name_manager()
|
||||
.get(&self.data.catalog_name, &self.data.object_name)
|
||||
.await?
|
||||
.with_context(|| FlowNotFoundSnafu {
|
||||
flow_name: &self.data.object_name,
|
||||
})?;
|
||||
|
||||
let flow_id = flow_name_value.flow_id();
|
||||
let flow_id = if let Some(flow_id) = self.data.flow_id {
|
||||
flow_id
|
||||
} else {
|
||||
self.context
|
||||
.flow_metadata_manager
|
||||
.flow_name_manager()
|
||||
.get(&self.data.catalog_name, &self.data.object_name)
|
||||
.await?
|
||||
.with_context(|| FlowNotFoundSnafu {
|
||||
flow_name: format_full_flow_name(
|
||||
&self.data.catalog_name,
|
||||
&self.data.object_name,
|
||||
),
|
||||
})?
|
||||
.flow_id()
|
||||
};
|
||||
let flow_info = self
|
||||
.context
|
||||
.flow_metadata_manager
|
||||
@@ -216,7 +224,7 @@ impl CommentOnProcedure {
|
||||
.get_raw(flow_id)
|
||||
.await?
|
||||
.with_context(|| FlowNotFoundSnafu {
|
||||
flow_name: &self.data.object_name,
|
||||
flow_name: format_full_flow_name(&self.data.catalog_name, &self.data.object_name),
|
||||
})?;
|
||||
|
||||
self.data.flow_id = Some(flow_id);
|
||||
@@ -411,17 +419,23 @@ impl Procedure for CommentOnProcedure {
|
||||
|
||||
let lock_key = match self.data.object_type {
|
||||
CommentObjectType::Table | CommentObjectType::Column => {
|
||||
vec![
|
||||
let mut lock_key = vec![
|
||||
CatalogLock::Read(catalog).into(),
|
||||
SchemaLock::read(catalog, schema).into(),
|
||||
TableNameLock::new(catalog, schema, &self.data.object_name).into(),
|
||||
]
|
||||
];
|
||||
if let Some(table_id) = self.data.table_id {
|
||||
lock_key.push(TableLock::Write(table_id).into());
|
||||
}
|
||||
lock_key.push(TableNameLock::new(catalog, schema, &self.data.object_name).into());
|
||||
lock_key
|
||||
}
|
||||
CommentObjectType::Flow => {
|
||||
vec![
|
||||
CatalogLock::Read(catalog).into(),
|
||||
FlowNameLock::new(catalog, &self.data.object_name).into(),
|
||||
]
|
||||
let mut lock_key = vec![CatalogLock::Read(catalog).into()];
|
||||
if let Some(flow_id) = self.data.flow_id {
|
||||
lock_key.push(FlowLock::Write(flow_id).into());
|
||||
}
|
||||
lock_key.push(FlowNameLock::new(catalog, &self.data.object_name).into());
|
||||
lock_key
|
||||
}
|
||||
};
|
||||
|
||||
@@ -466,6 +480,12 @@ pub struct CommentOnData {
|
||||
|
||||
impl CommentOnData {
|
||||
pub fn new(task: CommentOnTask) -> Self {
|
||||
let (table_id, flow_id) = match task.object_id {
|
||||
Some(CommentObjectId::Table(table_id)) => (Some(table_id), None),
|
||||
Some(CommentObjectId::Flow(flow_id)) => (None, Some(flow_id)),
|
||||
None => (None, None),
|
||||
};
|
||||
|
||||
Self {
|
||||
state: CommentOnState::Prepare,
|
||||
catalog_name: task.catalog_name,
|
||||
@@ -474,9 +494,9 @@ impl CommentOnData {
|
||||
object_name: task.object_name,
|
||||
column_name: task.column_name,
|
||||
comment: task.comment,
|
||||
table_id: None,
|
||||
table_id,
|
||||
table_info: None,
|
||||
flow_id: None,
|
||||
flow_id,
|
||||
flow_info: None,
|
||||
is_unchanged: false,
|
||||
}
|
||||
|
||||
@@ -574,9 +574,15 @@ impl DdlManager {
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn submit_comment_on_task(
|
||||
&self,
|
||||
comment_on_task: CommentOnTask,
|
||||
mut comment_on_task: CommentOnTask,
|
||||
) -> Result<(ProcedureId, Option<Output>)> {
|
||||
let context = self.create_context();
|
||||
comment_on_task
|
||||
.enrich_object_id(
|
||||
context.table_metadata_manager.table_name_manager(),
|
||||
context.flow_metadata_manager.flow_name_manager(),
|
||||
)
|
||||
.await?;
|
||||
let procedure = CommentOnProcedure::new(comment_on_task, context);
|
||||
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ use api::v1::{
|
||||
};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose;
|
||||
use common_catalog::{format_full_flow_name, format_full_table_name};
|
||||
use common_error::ext::BoxedError;
|
||||
use common_time::{DatabaseTimeToLive, Timestamp};
|
||||
use prost::Message;
|
||||
@@ -56,7 +57,11 @@ use crate::error::{
|
||||
self, ConvertTimeRangesSnafu, ExternalSnafu, InvalidSetDatabaseOptionSnafu,
|
||||
InvalidUnsetDatabaseOptionSnafu, Result,
|
||||
};
|
||||
use crate::flow_name::FlowName;
|
||||
use crate::instruction::CacheIdent;
|
||||
use crate::key::FlowId;
|
||||
use crate::key::flow::flow_name::FlowNameManager;
|
||||
use crate::key::table_name::{TableNameKey, TableNameManager};
|
||||
|
||||
/// Reserved query-context extension key for the frontend peer address that submitted a DDL request.
|
||||
pub const ORIGIN_FRONTEND_ADDR_EXTENSION_KEY: &str = "__greptime_origin_frontend.addr";
|
||||
@@ -1351,13 +1356,99 @@ pub enum CommentObjectType {
|
||||
}
|
||||
|
||||
impl CommentOnTask {
|
||||
pub fn table_ref(&self) -> TableReference<'_> {
|
||||
TableReference {
|
||||
catalog: &self.catalog_name,
|
||||
schema: &self.schema_name,
|
||||
table: &self.object_name,
|
||||
pub fn table_id(&self) -> Option<TableId> {
|
||||
match self.object_id.as_ref() {
|
||||
Some(CommentObjectId::Table(table_id)) => Some(*table_id),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn flow_id(&self) -> Option<FlowId> {
|
||||
match self.object_id.as_ref() {
|
||||
Some(CommentObjectId::Flow(flow_id)) => Some(*flow_id),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_table_id(&mut self, table_id: TableId) {
|
||||
self.object_id = Some(CommentObjectId::Table(table_id));
|
||||
}
|
||||
|
||||
fn set_flow_id(&mut self, flow_id: FlowId) {
|
||||
self.object_id = Some(CommentObjectId::Flow(flow_id));
|
||||
}
|
||||
|
||||
/// Returns the cache identifiers for the object being commented on.
|
||||
pub fn cache_idents(&self) -> Vec<CacheIdent> {
|
||||
match self.object_type {
|
||||
CommentObjectType::Table | CommentObjectType::Column => {
|
||||
let mut cache_idents = Vec::with_capacity(2);
|
||||
if let Some(CommentObjectId::Table(table_id)) = self.object_id.as_ref() {
|
||||
cache_idents.push(CacheIdent::TableId(*table_id));
|
||||
}
|
||||
cache_idents.push(CacheIdent::TableName(TableName {
|
||||
catalog_name: self.catalog_name.clone(),
|
||||
schema_name: self.schema_name.clone(),
|
||||
table_name: self.object_name.clone(),
|
||||
}));
|
||||
cache_idents
|
||||
}
|
||||
CommentObjectType::Flow => {
|
||||
let mut cache_idents = Vec::with_capacity(2);
|
||||
if let Some(CommentObjectId::Flow(flow_id)) = self.object_id.as_ref() {
|
||||
cache_idents.push(CacheIdent::FlowId(*flow_id));
|
||||
}
|
||||
cache_idents.push(CacheIdent::FlowName(FlowName {
|
||||
catalog_name: self.catalog_name.clone(),
|
||||
flow_name: self.object_name.clone(),
|
||||
}));
|
||||
cache_idents
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Enriches the `object_id` field of the `CommentOnTask`
|
||||
/// by looking up the corresponding table or flow ID using the provided managers.
|
||||
pub async fn enrich_object_id(
|
||||
&mut self,
|
||||
table_name_manager: &TableNameManager,
|
||||
flow_name_manager: &FlowNameManager,
|
||||
) -> Result<()> {
|
||||
match self.object_type {
|
||||
CommentObjectType::Table | CommentObjectType::Column => {
|
||||
let table_id = table_name_manager
|
||||
.get(TableNameKey::new(
|
||||
&self.catalog_name,
|
||||
&self.schema_name,
|
||||
&self.object_name,
|
||||
))
|
||||
.await?
|
||||
.with_context(|| error::TableNotFoundSnafu {
|
||||
table_name: format_full_table_name(
|
||||
&self.catalog_name,
|
||||
&self.schema_name,
|
||||
&self.object_name,
|
||||
),
|
||||
})?
|
||||
.table_id();
|
||||
|
||||
self.set_table_id(table_id);
|
||||
}
|
||||
CommentObjectType::Flow => {
|
||||
let flow_id = flow_name_manager
|
||||
.get(&self.catalog_name, &self.object_name)
|
||||
.await?
|
||||
.with_context(|| error::FlowNotFoundSnafu {
|
||||
flow_name: format_full_flow_name(&self.catalog_name, &self.object_name),
|
||||
})?
|
||||
.flow_id();
|
||||
|
||||
self.set_flow_id(flow_id);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Proto conversions for CommentObjectType
|
||||
|
||||
@@ -82,7 +82,7 @@ pub struct RemoveRegionFollowerRequest {
|
||||
pub struct GcRegionsRequest {
|
||||
pub region_ids: Vec<u64>,
|
||||
pub full_file_listing: bool,
|
||||
pub timeout: Duration,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -91,7 +91,7 @@ pub struct GcTableRequest {
|
||||
pub schema_name: String,
|
||||
pub table_name: String,
|
||||
pub full_file_listing: bool,
|
||||
pub timeout: Duration,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
|
||||
@@ -261,4 +261,26 @@ mod tests {
|
||||
};
|
||||
assert_eq!(datanode_wal_config, DatanodeWalConfig::Kafka(expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_kafka_wal_config_debug_redacts_password() {
|
||||
let config = MetasrvWalConfig::Kafka(MetasrvKafkaConfig {
|
||||
connection: KafkaConnectionConfig {
|
||||
sasl: Some(KafkaClientSasl {
|
||||
config: KafkaClientSaslConfig::Plain {
|
||||
username: "greptime".to_string(),
|
||||
password: "kafka-secret".to_string(),
|
||||
},
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let debug = format!("{config:#?}");
|
||||
|
||||
assert!(debug.contains("greptime"));
|
||||
assert!(debug.contains("<REDACTED>"));
|
||||
assert!(!debug.contains("kafka-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt;
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -57,7 +58,7 @@ pub struct KafkaClientSasl {
|
||||
pub config: KafkaClientSaslConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(tag = "type", rename_all = "SCREAMING-KEBAB-CASE")]
|
||||
pub enum KafkaClientSaslConfig {
|
||||
Plain {
|
||||
@@ -76,6 +77,28 @@ pub enum KafkaClientSaslConfig {
|
||||
},
|
||||
}
|
||||
|
||||
impl fmt::Debug for KafkaClientSaslConfig {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
KafkaClientSaslConfig::Plain { username, .. } => f
|
||||
.debug_struct("Plain")
|
||||
.field("username", username)
|
||||
.field("password", &"<REDACTED>")
|
||||
.finish(),
|
||||
KafkaClientSaslConfig::ScramSha256 { username, .. } => f
|
||||
.debug_struct("ScramSha256")
|
||||
.field("username", username)
|
||||
.field("password", &"<REDACTED>")
|
||||
.finish(),
|
||||
KafkaClientSaslConfig::ScramSha512 { username, .. } => f
|
||||
.debug_struct("ScramSha512")
|
||||
.field("username", username)
|
||||
.field("password", &"<REDACTED>")
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl KafkaClientSaslConfig {
|
||||
/// Converts to [`SaslConfig`].
|
||||
pub fn into_sasl_config(self) -> SaslConfig {
|
||||
|
||||
@@ -204,6 +204,8 @@ impl Instance {
|
||||
|
||||
let is_readonly_stmt = stmt.is_readonly();
|
||||
if should_track_statement_process(&stmt) {
|
||||
let catalog_name = query_ctx.current_catalog().to_string();
|
||||
let schema_name = query_ctx.current_schema();
|
||||
let slow_query_timer = if is_readonly_stmt {
|
||||
self.slow_query_options
|
||||
.enable
|
||||
@@ -212,6 +214,7 @@ impl Instance {
|
||||
.map(|event_recorder| {
|
||||
SlowQueryTimer::new(
|
||||
CatalogQueryStatement::Sql(stmt.clone()),
|
||||
schema_name.clone(),
|
||||
self.slow_query_options.threshold,
|
||||
self.slow_query_options.sample_ratio,
|
||||
self.slow_query_options.record_type,
|
||||
@@ -223,8 +226,8 @@ impl Instance {
|
||||
};
|
||||
|
||||
let ticket = self.process_manager.register_query(
|
||||
query_ctx.current_catalog().to_string(),
|
||||
vec![query_ctx.current_schema()],
|
||||
catalog_name,
|
||||
vec![schema_name],
|
||||
stmt.to_string(),
|
||||
query_ctx.conn_info().to_string(),
|
||||
Some(query_ctx.process_id()),
|
||||
@@ -662,6 +665,8 @@ impl Instance {
|
||||
|
||||
let plan_is_readonly = is_readonly_plan(&plan);
|
||||
let result = if should_track_plan_process(stmt.as_ref(), &plan) {
|
||||
let catalog_name = query_ctx.current_catalog().to_string();
|
||||
let schema_name = query_ctx.current_schema();
|
||||
let slow_query_timer = if plan_is_readonly {
|
||||
self.slow_query_options
|
||||
.enable
|
||||
@@ -670,6 +675,7 @@ impl Instance {
|
||||
.map(|event_recorder| {
|
||||
SlowQueryTimer::new(
|
||||
CatalogQueryStatement::Plan(query.clone()),
|
||||
schema_name.clone(),
|
||||
self.slow_query_options.threshold,
|
||||
self.slow_query_options.sample_ratio,
|
||||
self.slow_query_options.record_type,
|
||||
@@ -681,8 +687,8 @@ impl Instance {
|
||||
};
|
||||
|
||||
let ticket = self.process_manager.register_query(
|
||||
query_ctx.current_catalog().to_string(),
|
||||
vec![query_ctx.current_schema()],
|
||||
catalog_name,
|
||||
vec![schema_name],
|
||||
query,
|
||||
query_ctx.conn_info().to_string(),
|
||||
Some(query_ctx.process_id()),
|
||||
@@ -908,6 +914,7 @@ impl PrometheusHandler for Instance {
|
||||
.map(|event_recorder| {
|
||||
SlowQueryTimer::new(
|
||||
query_statement,
|
||||
query_ctx.current_schema(),
|
||||
self.slow_query_options.threshold,
|
||||
self.slow_query_options.sample_ratio,
|
||||
self.slow_query_options.record_type,
|
||||
|
||||
@@ -286,7 +286,7 @@ impl Inner {
|
||||
}),
|
||||
region_ids: request.region_ids,
|
||||
full_file_listing: request.full_file_listing,
|
||||
timeout_secs: timeout.as_secs() as u32,
|
||||
timeout_secs: gc_timeout_secs(timeout),
|
||||
};
|
||||
|
||||
let resp: GcRegionsResponse = self
|
||||
@@ -294,7 +294,9 @@ impl Inner {
|
||||
"gc_regions",
|
||||
move |mut client| {
|
||||
let mut req = Request::new(req.clone());
|
||||
req.set_timeout(timeout);
|
||||
if let Some(timeout) = timeout {
|
||||
req.set_timeout(timeout);
|
||||
}
|
||||
async move { client.gc_regions(req).await.map(|res| res.into_inner()) }
|
||||
},
|
||||
|resp: &GcRegionsResponse| &resp.header,
|
||||
@@ -323,7 +325,7 @@ impl Inner {
|
||||
schema_name: request.schema_name,
|
||||
table_name: request.table_name,
|
||||
full_file_listing: request.full_file_listing,
|
||||
timeout_secs: timeout.as_secs() as u32,
|
||||
timeout_secs: gc_timeout_secs(timeout),
|
||||
};
|
||||
|
||||
let resp: GcTableResponse = self
|
||||
@@ -331,7 +333,9 @@ impl Inner {
|
||||
"gc_table",
|
||||
move |mut client| {
|
||||
let mut req = Request::new(req.clone());
|
||||
req.set_timeout(timeout);
|
||||
if let Some(timeout) = timeout {
|
||||
req.set_timeout(timeout);
|
||||
}
|
||||
async move { client.gc_table(req).await.map(|res| res.into_inner()) }
|
||||
},
|
||||
|resp: &GcTableResponse| &resp.header,
|
||||
@@ -413,6 +417,12 @@ impl Inner {
|
||||
}
|
||||
}
|
||||
|
||||
fn gc_timeout_secs(timeout: Option<Duration>) -> u32 {
|
||||
timeout
|
||||
.map(|timeout| timeout.as_secs().max(1).try_into().unwrap_or(u32::MAX))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -438,8 +448,18 @@ mod tests {
|
||||
use tonic::codec::CompressionEncoding;
|
||||
use tonic::{Request, Response, Status};
|
||||
|
||||
use super::gc_timeout_secs;
|
||||
use crate::client::MetaClientBuilder;
|
||||
|
||||
#[test]
|
||||
fn test_gc_timeout_secs() {
|
||||
assert_eq!(gc_timeout_secs(None), 0);
|
||||
assert_eq!(gc_timeout_secs(Some(Duration::from_millis(1))), 1);
|
||||
assert_eq!(gc_timeout_secs(Some(Duration::from_millis(999))), 1);
|
||||
assert_eq!(gc_timeout_secs(Some(Duration::from_secs(1))), 1);
|
||||
assert_eq!(gc_timeout_secs(Some(Duration::from_secs(10))), 10);
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MockHeartbeat {
|
||||
leader_addr: String,
|
||||
|
||||
@@ -268,7 +268,7 @@ impl procedure_service_server::ProcedureService for Metasrv {
|
||||
.handle_gc_regions(MetaGcRegionsRequest {
|
||||
region_ids,
|
||||
full_file_listing,
|
||||
timeout: Duration::from_secs(timeout_secs as u64),
|
||||
timeout: Self::normalize_gc_timeout(Duration::from_secs(timeout_secs as u64)),
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -295,7 +295,7 @@ impl procedure_service_server::ProcedureService for Metasrv {
|
||||
schema_name,
|
||||
table_name,
|
||||
full_file_listing,
|
||||
timeout: Duration::from_secs(timeout_secs as u64),
|
||||
timeout: Self::normalize_gc_timeout(Duration::from_secs(timeout_secs as u64)),
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -356,9 +356,8 @@ impl Metasrv {
|
||||
&self,
|
||||
region_ids: Vec<RegionId>,
|
||||
full_file_listing: bool,
|
||||
timeout: Duration,
|
||||
timeout: Option<Duration>,
|
||||
) -> error::Result<GcResponse> {
|
||||
let timeout = Self::normalize_gc_timeout(timeout);
|
||||
let gc_ticker = self.gc_ticker().context(error::UnexpectedSnafu {
|
||||
violated: "GC ticker not available".to_string(),
|
||||
})?;
|
||||
|
||||
@@ -1297,7 +1297,9 @@ impl CacheManagerBuilder {
|
||||
|
||||
fn meta_cache_weight(k: &SstMetaKey, v: &Arc<CachedSstMeta>) -> u32 {
|
||||
// We ignore the size of `Arc`.
|
||||
(k.estimated_size() + parquet_meta_size(&v.parquet_metadata) + v.region_metadata_weight) as u32
|
||||
let size =
|
||||
k.estimated_size() + parquet_meta_size(&v.parquet_metadata) + v.region_metadata_weight;
|
||||
u32::try_from(size).unwrap_or(u32::MAX)
|
||||
}
|
||||
|
||||
fn vector_cache_weight(_k: &(ConcreteDataType, Value), v: &VectorRef) -> u32 {
|
||||
@@ -1884,6 +1886,19 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_meta_cache_weight_saturates_on_overflow() {
|
||||
let region_metadata = Arc::new(wide_region_metadata(1));
|
||||
let metadata = sst_parquet_meta_with_region_metadata(region_metadata.clone());
|
||||
let mut cached =
|
||||
CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap();
|
||||
cached.region_metadata_weight = u32::MAX as usize + 1;
|
||||
let cached = Arc::new(cached);
|
||||
let key = SstMetaKey(region_metadata.region_id, FileId::random());
|
||||
|
||||
assert_eq!(u32::MAX, meta_cache_weight(&key, &cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_repeated_vector_cache() {
|
||||
let cache = CacheManager::builder().vector_cache_size(4096).build();
|
||||
|
||||
151
src/mito2/src/cache/cache_size.rs
vendored
151
src/mito2/src/cache/cache_size.rs
vendored
@@ -14,119 +14,52 @@
|
||||
|
||||
//! Cache size of different cache value.
|
||||
|
||||
use std::mem;
|
||||
|
||||
use parquet::basic::ColumnOrder;
|
||||
use parquet::file::metadata::{
|
||||
FileMetaData, KeyValue, ParquetColumnIndex, ParquetMetaData, ParquetOffsetIndex,
|
||||
RowGroupMetaData,
|
||||
};
|
||||
use parquet::file::page_index::column_index::ColumnIndexMetaData as Index;
|
||||
use parquet::file::page_index::offset_index::PageLocation;
|
||||
use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor, Type};
|
||||
use parquet::file::metadata::ParquetMetaData;
|
||||
|
||||
/// Returns estimated size of [ParquetMetaData].
|
||||
pub fn parquet_meta_size(meta: &ParquetMetaData) -> usize {
|
||||
// struct size
|
||||
let mut size = mem::size_of::<ParquetMetaData>();
|
||||
// file_metadata
|
||||
size += file_meta_heap_size(meta.file_metadata());
|
||||
// row_groups
|
||||
size += meta
|
||||
.row_groups()
|
||||
.iter()
|
||||
.map(row_group_meta_heap_size)
|
||||
.sum::<usize>();
|
||||
// column_index
|
||||
size += meta
|
||||
.column_index()
|
||||
.map(parquet_column_index_heap_size)
|
||||
.unwrap_or(0);
|
||||
// offset_index
|
||||
size += meta
|
||||
.offset_index()
|
||||
.map(parquet_offset_index_heap_size)
|
||||
.unwrap_or(0);
|
||||
|
||||
size
|
||||
meta.memory_size()
|
||||
}
|
||||
|
||||
/// Returns estimated size of [FileMetaData] allocated from heap.
|
||||
fn file_meta_heap_size(meta: &FileMetaData) -> usize {
|
||||
// created_by
|
||||
let mut size = meta.created_by().map(|s| s.len()).unwrap_or(0);
|
||||
// key_value_metadata
|
||||
size += meta
|
||||
.key_value_metadata()
|
||||
.map(|kvs| {
|
||||
kvs.iter()
|
||||
.map(|kv| {
|
||||
kv.key.len()
|
||||
+ kv.value.as_ref().map(|v| v.len()).unwrap_or(0)
|
||||
+ mem::size_of::<KeyValue>()
|
||||
})
|
||||
.sum()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
// schema_descr (It's a ptr so we also add size of SchemaDescriptor).
|
||||
size += mem::size_of::<SchemaDescriptor>();
|
||||
size += schema_descr_heap_size(meta.schema_descr());
|
||||
// column_orders
|
||||
size += meta
|
||||
.column_orders()
|
||||
.map(|orders| orders.len() * mem::size_of::<ColumnOrder>())
|
||||
.unwrap_or(0);
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
size
|
||||
}
|
||||
|
||||
/// Returns estimated size of [SchemaDescriptor] allocated from heap.
|
||||
fn schema_descr_heap_size(descr: &SchemaDescriptor) -> usize {
|
||||
// schema
|
||||
let mut size = mem::size_of::<Type>();
|
||||
// leaves
|
||||
size += descr
|
||||
.columns()
|
||||
.iter()
|
||||
.map(|descr| mem::size_of::<ColumnDescriptor>() + column_descr_heap_size(descr))
|
||||
.sum::<usize>();
|
||||
// leaf_to_base
|
||||
size += descr.num_columns() * mem::size_of::<usize>();
|
||||
|
||||
size
|
||||
}
|
||||
|
||||
/// Returns estimated size of [ColumnDescriptor] allocated from heap.
|
||||
fn column_descr_heap_size(descr: &ColumnDescriptor) -> usize {
|
||||
descr.path().parts().iter().map(|s| s.len()).sum()
|
||||
}
|
||||
|
||||
/// Returns estimated size of [ColumnDescriptor] allocated from heap.
|
||||
fn row_group_meta_heap_size(meta: &RowGroupMetaData) -> usize {
|
||||
mem::size_of_val(meta.columns())
|
||||
}
|
||||
|
||||
/// Returns estimated size of [ParquetColumnIndex] allocated from heap.
|
||||
fn parquet_column_index_heap_size(column_index: &ParquetColumnIndex) -> usize {
|
||||
column_index
|
||||
.iter()
|
||||
.map(|row_group| row_group.len() * mem::size_of::<Index>() + mem::size_of_val(row_group))
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Returns estimated size of [ParquetOffsetIndex] allocated from heap.
|
||||
fn parquet_offset_index_heap_size(offset_index: &ParquetOffsetIndex) -> usize {
|
||||
offset_index
|
||||
.iter()
|
||||
.map(|row_group| {
|
||||
row_group
|
||||
.iter()
|
||||
.map(|column| {
|
||||
column.page_locations.len() * mem::size_of::<PageLocation>()
|
||||
+ mem::size_of_val(column)
|
||||
})
|
||||
.sum::<usize>()
|
||||
+ mem::size_of_val(row_group)
|
||||
})
|
||||
.sum()
|
||||
use parquet::basic::{Repetition, Type as PhysicalType};
|
||||
use parquet::file::metadata::{ColumnIndexBuilder, FileMetaData, ParquetMetaDataBuilder};
|
||||
use parquet::schema::types::{SchemaDescriptor, Type as SchemaType};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parquet_meta_size_counts_byte_array_column_index_buffers() {
|
||||
let field = Arc::new(
|
||||
SchemaType::primitive_type_builder("tag", PhysicalType::BYTE_ARRAY)
|
||||
.with_repetition(Repetition::OPTIONAL)
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
let schema = Arc::new(
|
||||
SchemaType::group_type_builder("schema")
|
||||
.with_fields(vec![field])
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
let schema_descr = Arc::new(SchemaDescriptor::new(schema));
|
||||
let file_metadata = FileMetaData::new(2, 3, None, None, schema_descr, None);
|
||||
|
||||
let mut column_index = ColumnIndexBuilder::new(PhysicalType::BYTE_ARRAY);
|
||||
for page in 0..3u8 {
|
||||
column_index.append(false, vec![page; 4096], vec![page + 1; 4096], 0);
|
||||
}
|
||||
let metadata = ParquetMetaDataBuilder::new(file_metadata)
|
||||
.set_column_index(Some(vec![vec![column_index.build().unwrap()]]))
|
||||
.build();
|
||||
|
||||
let min_max_bytes = 3 * 4096 * 2;
|
||||
assert!(
|
||||
parquet_meta_size(&metadata) >= min_max_bytes,
|
||||
"metadata size should include the byte-array page-index min/max buffers"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ use common_telemetry::tracing::Instrument as _;
|
||||
use common_telemetry::{debug, error, info, warn};
|
||||
use common_time::Timestamp;
|
||||
use itertools::Itertools;
|
||||
use object_store::{Entry, Lister};
|
||||
use object_store::{Entry, ErrorKind, Lister};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use snafu::{ResultExt as _, ensure};
|
||||
use store_api::storage::{FileId, FileRef, FileRefsManifest, GcReport, IndexVersion, RegionId};
|
||||
@@ -308,7 +308,14 @@ impl LocalGcWorker {
|
||||
.iter()
|
||||
.filter_map(|f| f.index_version().map(|v| (f.file_id(), v)))
|
||||
.collect_vec();
|
||||
deleted_files.insert(*region_id, files.into_iter().map(|f| f.file_id()).collect());
|
||||
let data_files = files
|
||||
.into_iter()
|
||||
.filter_map(|f| match f {
|
||||
RemovedFile::File(file_id, _) => Some(file_id),
|
||||
RemovedFile::Index(_, _) => None,
|
||||
})
|
||||
.collect();
|
||||
deleted_files.insert(*region_id, data_files);
|
||||
deleted_indexes.insert(*region_id, index_files);
|
||||
processed_regions.insert(*region_id);
|
||||
debug!(
|
||||
@@ -671,22 +678,72 @@ impl LocalGcWorker {
|
||||
})?;
|
||||
let lister_cnt = listers.len();
|
||||
|
||||
// Step 2: Concurrently list all files in the region directory
|
||||
let all_entries = self
|
||||
// Step 2: Concurrently list all parquet files in the region root directory
|
||||
let mut all_entries = self
|
||||
.list_region_files_concurrent(listers)
|
||||
.await
|
||||
.inspect_err(|_| {
|
||||
GC_ERRORS_TOTAL.with_label_values(&["list_failed"]).inc();
|
||||
})?;
|
||||
let cnt = all_entries.len();
|
||||
let root_cnt = all_entries.len();
|
||||
|
||||
// Step 2b: Flat-list region_dir/index/ for puffin files.
|
||||
// This is NOT a recursive listing — we only list the index/
|
||||
// subdirectory to avoid scanning nested dirs/staging/blob/cache.
|
||||
let index_entries = self
|
||||
.list_region_index_files(region_id)
|
||||
.await
|
||||
.inspect_err(|_| {
|
||||
GC_ERRORS_TOTAL.with_label_values(&["list_failed"]).inc();
|
||||
})?;
|
||||
let index_cnt = index_entries.len();
|
||||
all_entries.extend(index_entries);
|
||||
info!(
|
||||
"gc: full listing mode cost {} secs using {lister_cnt} lister for {cnt} files in region {}.",
|
||||
"gc: full listing mode cost {} secs using {lister_cnt} lister for root={root_cnt} index={index_cnt} files in region {}.",
|
||||
start.elapsed().as_secs_f64(),
|
||||
region_id
|
||||
);
|
||||
Ok(all_entries)
|
||||
}
|
||||
|
||||
/// Flat-list puffin files from `region_dir/index/`.
|
||||
/// If the index directory does not exist, returns an empty vec without error.
|
||||
/// Only `.puffin` files (not subdirectories) are included.
|
||||
async fn list_region_index_files(&self, region_id: RegionId) -> Result<Vec<Entry>> {
|
||||
let region_dir = self.access_layer.build_region_dir(region_id);
|
||||
let index_dir = object_store::util::join_dir(®ion_dir, "index");
|
||||
|
||||
let mut lister = match self
|
||||
.access_layer
|
||||
.object_store()
|
||||
.lister_with(&index_dir)
|
||||
.await
|
||||
{
|
||||
Ok(l) => l,
|
||||
Err(e) if e.kind() == ErrorKind::NotFound => {
|
||||
// Index dir may not exist — that's fine, just log and return empty.
|
||||
// object-store backends (especially filesystem) may error on
|
||||
// non-existent directories.
|
||||
debug!(
|
||||
"Index directory not found for region {}: {}. Treating as empty.",
|
||||
region_id, e
|
||||
);
|
||||
return Ok(vec![]);
|
||||
}
|
||||
Err(e) => return Err(e).context(OpenDalSnafu),
|
||||
};
|
||||
|
||||
let mut entries = Vec::new();
|
||||
while let Some(entry) = lister.next().await {
|
||||
let entry = entry.context(OpenDalSnafu)?;
|
||||
if entry.metadata().is_file() && entry.name().ends_with(".puffin") {
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Concurrently list all files in the region directory using the provided listers.
|
||||
/// Returns a vector of all file entries found across all partitions.
|
||||
async fn list_region_files_concurrent(
|
||||
@@ -710,7 +767,8 @@ impl LocalGcWorker {
|
||||
true
|
||||
}
|
||||
}
|
||||
// entry went wrong, log and skip it
|
||||
// Entry went wrong. Keep listing so the error can be propagated below
|
||||
// instead of returning a partial listing as success.
|
||||
Err(err) => {
|
||||
warn!("Failed to list entry: {}", err);
|
||||
true
|
||||
@@ -747,7 +805,9 @@ impl LocalGcWorker {
|
||||
// Collect all entries from the channel
|
||||
let mut all_entries = vec![];
|
||||
while let Some(stream) = rx.recv().await {
|
||||
all_entries.extend(stream.into_iter().filter_map(Result::ok));
|
||||
for entry in stream {
|
||||
all_entries.push(entry.context(OpenDalSnafu)?);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(all_entries)
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Pruner for parallel file pruning across scanner partitions.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -117,6 +117,35 @@ impl PartitionPruner {
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
/// Checks whether the file range at `index` can be skipped because the
|
||||
/// current predicate definitively prunes it at manifest level (no I/O).
|
||||
///
|
||||
/// Returns `true` if the range was skipped. When skipped, this method
|
||||
/// balances the pruner's per-file reference count and merges the resulting
|
||||
/// reader metrics into `part_metrics`.
|
||||
///
|
||||
/// Uses shared per-file state so repeated row groups can skip cheaply after
|
||||
/// the first manifest-prune decision.
|
||||
pub fn try_skip_manifest_pruned_file_range(
|
||||
&self,
|
||||
index: RowGroupIndex,
|
||||
part_metrics: &PartitionMetrics,
|
||||
) -> bool {
|
||||
let Some(file_index) = self.file_index(index) else {
|
||||
return false;
|
||||
};
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
let pruned = self
|
||||
.pruner
|
||||
.inner
|
||||
.try_mark_manifest_pruned(file_index, &mut reader_metrics);
|
||||
if pruned {
|
||||
self.pruner.skip_file_range(index, &mut reader_metrics);
|
||||
part_metrics.merge_reader_metrics(&reader_metrics, None);
|
||||
}
|
||||
pruned
|
||||
}
|
||||
|
||||
/// Pre-fetches upcoming files starting from the given position.
|
||||
fn prefetch_upcoming_files(&self, current_pos: usize, partition_metrics: &PartitionMetrics) {
|
||||
let start = current_pos + 1;
|
||||
@@ -139,6 +168,14 @@ impl PartitionPruner {
|
||||
.copied()
|
||||
.unwrap_or(PreFilterMode::SkipFields)
|
||||
}
|
||||
|
||||
fn file_index(&self, index: RowGroupIndex) -> Option<usize> {
|
||||
self.pruner
|
||||
.inner
|
||||
.stream_ctx
|
||||
.is_file_range_index(index)
|
||||
.then(|| index.index - self.pruner.inner.stream_ctx.input.num_memtables())
|
||||
}
|
||||
}
|
||||
|
||||
/// A pruner that prunes files for all partitions of a scanner.
|
||||
@@ -155,6 +192,40 @@ struct PrunerInner {
|
||||
file_entries: Vec<Mutex<FileBuilderEntry>>,
|
||||
/// StreamContext containing all context needed for pruning.
|
||||
stream_ctx: Arc<StreamContext>,
|
||||
/// Positive manifest-prune cache shared across all scan partitions.
|
||||
///
|
||||
/// SAFETY: cached positives are valid because dynamic filters only tighten;
|
||||
/// negative decisions are not cached. Reset by `add_partition_ranges()` for
|
||||
/// each fresh batch of partition ranges.
|
||||
manifest_pruned_files: Vec<AtomicBool>,
|
||||
}
|
||||
|
||||
impl PrunerInner {
|
||||
/// Checks whether manifest-level pruning proves this file is empty given the
|
||||
/// current predicate. If true, CAS the shared cache from false→true and
|
||||
/// record `files_time_range_pruned` in `reader_metrics`.
|
||||
///
|
||||
/// Returns `true` if already cached or newly proven pruned.
|
||||
fn try_mark_manifest_pruned(
|
||||
&self,
|
||||
file_index: usize,
|
||||
reader_metrics: &mut ReaderMetrics,
|
||||
) -> bool {
|
||||
if self.manifest_pruned_files[file_index].load(Ordering::Relaxed) {
|
||||
return true;
|
||||
}
|
||||
let file = &self.stream_ctx.input.files[file_index];
|
||||
if !self.stream_ctx.input.can_manifest_prune_file(file) {
|
||||
return false;
|
||||
}
|
||||
if self.manifest_pruned_files[file_index]
|
||||
.compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
reader_metrics.filter_metrics.files_time_range_pruned += 1;
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-file state tracking.
|
||||
@@ -196,6 +267,8 @@ impl Pruner {
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let manifest_pruned_files: Vec<AtomicBool> =
|
||||
(0..num_files).map(|_| AtomicBool::new(false)).collect();
|
||||
// Create channels and collect senders
|
||||
let mut worker_senders = Vec::with_capacity(num_workers);
|
||||
let mut receivers = Vec::with_capacity(num_workers);
|
||||
@@ -209,6 +282,7 @@ impl Pruner {
|
||||
num_workers,
|
||||
file_entries,
|
||||
stream_ctx,
|
||||
manifest_pruned_files,
|
||||
});
|
||||
|
||||
// Spawn worker tasks with their receivers
|
||||
@@ -225,8 +299,14 @@ impl Pruner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds reference counts for all partitions' ranges.
|
||||
/// Adds reference counts for all partitions' ranges and resets the full
|
||||
/// manifest-prune cache so that dynamic-filter updates are visible to the
|
||||
/// fresh scan.
|
||||
pub fn add_partition_ranges(&self, partition_ranges: &[PartitionRange]) {
|
||||
for pruned in &self.inner.manifest_pruned_files {
|
||||
pruned.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
// Add reference counts for each partition range
|
||||
let num_memtables = self.inner.stream_ctx.input.num_memtables();
|
||||
for part_range in partition_ranges {
|
||||
@@ -277,6 +357,19 @@ impl Pruner {
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
/// Skips a file range that has been pruned before entering the file pruner.
|
||||
///
|
||||
/// This keeps the pruner's per-file reference counts balanced with
|
||||
/// `add_partition_ranges()`. It may also clear a cached builder when this was the
|
||||
/// last remaining range for the file.
|
||||
pub fn skip_file_range(&self, index: RowGroupIndex, reader_metrics: &mut ReaderMetrics) {
|
||||
if !self.inner.stream_ctx.is_file_range_index(index) {
|
||||
return;
|
||||
}
|
||||
let file_index = index.index - self.inner.stream_ctx.input.num_memtables();
|
||||
self.decrement_and_maybe_clear(file_index, reader_metrics);
|
||||
}
|
||||
|
||||
/// Gets or creates the FileRangeBuilder for a file.
|
||||
async fn get_file_builder(
|
||||
&self,
|
||||
@@ -374,25 +467,33 @@ impl Pruner {
|
||||
pre_filter_mode: PreFilterMode,
|
||||
reader_metrics: &mut ReaderMetrics,
|
||||
) -> Result<Arc<FileRangeBuilder>> {
|
||||
// Check manifest-level prune first (shared cache, no I/O).
|
||||
if self
|
||||
.inner
|
||||
.try_mark_manifest_pruned(file_index, reader_metrics)
|
||||
{
|
||||
let arc_builder = Arc::new(FileRangeBuilder::default());
|
||||
// Do NOT cache an empty manifest-pruned builder; the cache flag
|
||||
// already records the decision.
|
||||
return Ok(arc_builder);
|
||||
}
|
||||
|
||||
let file = &self.inner.stream_ctx.input.files[file_index];
|
||||
let predicate = self.inner.stream_ctx.input.predicate_for_file(file);
|
||||
let builder = self
|
||||
.inner
|
||||
.stream_ctx
|
||||
.input
|
||||
.prune_file(file, pre_filter_mode, reader_metrics)
|
||||
.prune_file_after_manifest_check(file, pre_filter_mode, predicate, reader_metrics)
|
||||
.await?;
|
||||
|
||||
let arc_builder = Arc::new(builder);
|
||||
|
||||
// Caches the builder
|
||||
// Caches the builder only if the file still has remaining ranges.
|
||||
// `skip_file_range` may have already consumed all ranges for this file.
|
||||
{
|
||||
let mut entry = self.inner.file_entries[file_index].lock().unwrap();
|
||||
if entry.builder.is_none() {
|
||||
reader_metrics.metadata_mem_size += arc_builder.memory_size() as isize;
|
||||
reader_metrics.num_range_builders += 1;
|
||||
entry.builder = Some(arc_builder.clone());
|
||||
PRUNER_ACTIVE_BUILDERS.inc();
|
||||
}
|
||||
cache_builder_if_needed(&mut entry, &arc_builder, reader_metrics);
|
||||
}
|
||||
|
||||
Ok(arc_builder)
|
||||
@@ -444,7 +545,6 @@ impl Pruner {
|
||||
}
|
||||
worker_cache_miss += 1;
|
||||
|
||||
// Do the actual pruning (outside lock)
|
||||
let file = &inner.stream_ctx.input.files[file_index];
|
||||
pruned_files.push(file.file_id().file_id());
|
||||
let explain_verbose = partition_metrics
|
||||
@@ -455,24 +555,45 @@ impl Pruner {
|
||||
filter_metrics: new_filter_metrics(explain_verbose),
|
||||
..Default::default()
|
||||
};
|
||||
let result = inner
|
||||
.stream_ctx
|
||||
.input
|
||||
.prune_file(file, pre_filter_mode, &mut metrics)
|
||||
.await;
|
||||
|
||||
// Check manifest-level prune first (shared cache, no I/O).
|
||||
let result = if inner.try_mark_manifest_pruned(file_index, &mut metrics) {
|
||||
// Manifest-level pruning proved the file empty — produce a
|
||||
// default builder without reading any parquet metadata.
|
||||
Ok(FileRangeBuilder::default())
|
||||
} else {
|
||||
let predicate = inner.stream_ctx.input.predicate_for_file(file);
|
||||
inner
|
||||
.stream_ctx
|
||||
.input
|
||||
.prune_file_after_manifest_check(file, pre_filter_mode, predicate, &mut metrics)
|
||||
.await
|
||||
};
|
||||
|
||||
// Update state and notify waiters
|
||||
let mut entry = inner.file_entries[file_index].lock().unwrap();
|
||||
match result {
|
||||
Ok(builder) => {
|
||||
let arc_builder = Arc::new(builder);
|
||||
entry.builder = Some(arc_builder.clone());
|
||||
PRUNER_ACTIVE_BUILDERS.inc();
|
||||
let is_background = response_tx.is_none();
|
||||
|
||||
// Only cache the builder if the file still has remaining ranges.
|
||||
// If remaining_ranges == 0, a concurrent `skip_file_range` (e.g. from a
|
||||
// dynamic filter tightening via manifest-prune fast-skip) already consumed
|
||||
// all ranges and may have cleared a previously cached builder.
|
||||
// Skip caching manifest-pruned empty builders; the cache flag is enough.
|
||||
let did_cache =
|
||||
if inner.manifest_pruned_files[file_index].load(Ordering::Relaxed) {
|
||||
false
|
||||
} else {
|
||||
cache_builder_if_needed(&mut entry, &arc_builder, &mut metrics)
|
||||
};
|
||||
|
||||
// Notify all waiters
|
||||
for waiter in entry.waiters.drain(..) {
|
||||
let _ = waiter.send(Ok(arc_builder.clone()));
|
||||
}
|
||||
// Always respond to foreground caller, even if we did not cache.
|
||||
if let Some(response_tx) = response_tx {
|
||||
let _ = response_tx.send(Ok(arc_builder));
|
||||
}
|
||||
@@ -485,8 +606,13 @@ impl Pruner {
|
||||
metrics
|
||||
);
|
||||
|
||||
// Merge metrics to partition if provided
|
||||
if let Some(part_metrics) = &partition_metrics {
|
||||
// Merge metrics if this is a foreground request, or if the builder
|
||||
// was cached. Skip stale per-file metrics
|
||||
// for background requests that completed after the file was already
|
||||
// fully skipped.
|
||||
if (!is_background || did_cache)
|
||||
&& let Some(part_metrics) = &partition_metrics
|
||||
{
|
||||
let per_file_metrics = if part_metrics.explain_verbose() {
|
||||
let file_id = file.file_id();
|
||||
let mut map = HashMap::new();
|
||||
@@ -525,3 +651,460 @@ impl Pruner {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Pruner {
|
||||
/// Returns the remaining range count for a file (test-only).
|
||||
fn test_remaining_ranges(&self, file_index: usize) -> usize {
|
||||
self.inner.file_entries[file_index]
|
||||
.lock()
|
||||
.unwrap()
|
||||
.remaining_ranges
|
||||
}
|
||||
|
||||
/// Returns whether a cached builder exists for a file (test-only).
|
||||
fn test_has_builder(&self, file_index: usize) -> bool {
|
||||
self.inner.file_entries[file_index]
|
||||
.lock()
|
||||
.unwrap()
|
||||
.builder
|
||||
.is_some()
|
||||
}
|
||||
|
||||
/// Returns the manifest-pruned flag for a file (test-only).
|
||||
fn test_is_manifest_pruned(&self, file_index: usize) -> bool {
|
||||
self.inner.manifest_pruned_files[file_index].load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Clears a cached builder for a file, simulating stale cleanup (test-only).
|
||||
#[allow(dead_code)]
|
||||
fn test_clear_builder(&self, file_index: usize) {
|
||||
let mut entry = self.inner.file_entries[file_index].lock().unwrap();
|
||||
if entry.builder.take().is_some() {
|
||||
PRUNER_ACTIVE_BUILDERS.dec();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if a freshly pruned builder should be cached for this file.
|
||||
fn should_cache_builder(entry: &FileBuilderEntry) -> bool {
|
||||
entry.builder.is_none() && entry.remaining_ranges > 0
|
||||
}
|
||||
|
||||
/// Caches a freshly pruned builder if the file still has remaining ranges, and
|
||||
/// records the corresponding builder memory/count deltas for verbose metrics.
|
||||
fn cache_builder_if_needed(
|
||||
entry: &mut FileBuilderEntry,
|
||||
builder: &Arc<FileRangeBuilder>,
|
||||
reader_metrics: &mut ReaderMetrics,
|
||||
) -> bool {
|
||||
if should_cache_builder(entry) {
|
||||
reader_metrics.metadata_mem_size += builder.memory_size() as isize;
|
||||
reader_metrics.num_range_builders += 1;
|
||||
entry.builder = Some(builder.clone());
|
||||
PRUNER_ACTIVE_BUILDERS.inc();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use common_time::Timestamp;
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_expr::{Expr, col, lit};
|
||||
use store_api::region_engine::PartitionRange;
|
||||
use store_api::storage::{FileId, RegionId};
|
||||
|
||||
use super::*;
|
||||
use crate::read::flat_projection::FlatProjectionMapper;
|
||||
use crate::read::range::RowGroupIndex;
|
||||
use crate::read::scan_region::{PredicateGroup, ScanInput};
|
||||
use crate::read::scan_util::PartitionMetrics;
|
||||
use crate::sst::file::{FileHandle, FileMeta};
|
||||
use crate::sst::parquet::reader::ReaderMetrics;
|
||||
use crate::test_util::memtable_util::metadata_with_primary_key;
|
||||
use crate::test_util::new_noop_file_purger;
|
||||
use crate::test_util::scheduler_util::SchedulerEnv;
|
||||
|
||||
async fn make_test_pruner(num_files: usize) -> (SchedulerEnv, Arc<Pruner>) {
|
||||
let env = SchedulerEnv::new().await;
|
||||
let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
|
||||
let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
|
||||
|
||||
let files: Vec<FileHandle> = (0..num_files)
|
||||
.map(|_| {
|
||||
let meta = FileMeta {
|
||||
region_id: RegionId::new(123, 456),
|
||||
file_id: FileId::random(),
|
||||
time_range: (
|
||||
Timestamp::new_millisecond(0),
|
||||
Timestamp::new_millisecond(1000),
|
||||
),
|
||||
num_row_groups: 1,
|
||||
num_rows: 1024,
|
||||
level: 0,
|
||||
..Default::default()
|
||||
};
|
||||
FileHandle::new(meta, new_noop_file_purger())
|
||||
})
|
||||
.collect();
|
||||
|
||||
let input = ScanInput::new(env.access_layer.clone(), mapper)
|
||||
.with_files(files)
|
||||
.with_append_mode(true);
|
||||
let stream_ctx = Arc::new(StreamContext::unordered_scan_ctx(input));
|
||||
let pruner = Arc::new(Pruner::new(stream_ctx, 1));
|
||||
(env, pruner)
|
||||
}
|
||||
|
||||
/// Builds a minimal `PartitionRange` that references `file_index`.
|
||||
/// `add_partition_ranges` will look up `stream_ctx.ranges[identifier]`
|
||||
/// and find `row_group_indices[0] == RowGroupIndex { index: file_index,
|
||||
/// row_group_index: 0 }` because `unordered_scan_ranges` with
|
||||
/// `num_row_groups=1` produces one range per file.
|
||||
fn file_partition_range(file_index: usize) -> PartitionRange {
|
||||
PartitionRange {
|
||||
start: Timestamp::new_millisecond(0),
|
||||
end: Timestamp::new_millisecond(1001),
|
||||
num_rows: 1024,
|
||||
identifier: file_index,
|
||||
}
|
||||
}
|
||||
|
||||
async fn make_test_pruner_with_predicate(
|
||||
num_files: usize,
|
||||
row_groups_per_file: u64,
|
||||
predicate_exprs: &[Expr],
|
||||
) -> (SchedulerEnv, Arc<Pruner>) {
|
||||
let env = SchedulerEnv::new().await;
|
||||
let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
|
||||
let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
|
||||
let predicate = PredicateGroup::new(&metadata, predicate_exprs).unwrap();
|
||||
|
||||
let files: Vec<FileHandle> = (0..num_files)
|
||||
.map(|_| {
|
||||
let meta = FileMeta {
|
||||
region_id: RegionId::new(123, 456),
|
||||
file_id: FileId::random(),
|
||||
time_range: (
|
||||
Timestamp::new_millisecond(0),
|
||||
Timestamp::new_millisecond(1000),
|
||||
),
|
||||
num_row_groups: row_groups_per_file,
|
||||
num_rows: row_groups_per_file * 1024,
|
||||
level: 0,
|
||||
..Default::default()
|
||||
};
|
||||
FileHandle::new(meta, new_noop_file_purger())
|
||||
})
|
||||
.collect();
|
||||
|
||||
let input = ScanInput::new(env.access_layer.clone(), mapper)
|
||||
.with_files(files)
|
||||
.with_predicate(predicate)
|
||||
.with_append_mode(true);
|
||||
let stream_ctx = Arc::new(StreamContext::unordered_scan_ctx(input));
|
||||
let pruner = Arc::new(Pruner::new(stream_ctx, 1));
|
||||
(env, pruner)
|
||||
}
|
||||
|
||||
fn make_partition_metrics() -> PartitionMetrics {
|
||||
let metrics_set = ExecutionPlanMetricsSet::new();
|
||||
PartitionMetrics::new(
|
||||
RegionId::new(123, 456),
|
||||
0,
|
||||
"test",
|
||||
Instant::now(),
|
||||
false,
|
||||
&metrics_set,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_cache_builder_when_ranges_remain() {
|
||||
let entry = FileBuilderEntry {
|
||||
builder: None,
|
||||
remaining_ranges: 3,
|
||||
waiters: Vec::new(),
|
||||
};
|
||||
assert!(should_cache_builder(&entry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_not_cache_builder_when_no_ranges_remain() {
|
||||
let entry = FileBuilderEntry {
|
||||
builder: None,
|
||||
remaining_ranges: 0,
|
||||
waiters: Vec::new(),
|
||||
};
|
||||
assert!(!should_cache_builder(&entry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_not_cache_builder_when_already_cached() {
|
||||
let entry = FileBuilderEntry {
|
||||
builder: Some(Arc::new(FileRangeBuilder::default())),
|
||||
remaining_ranges: 1,
|
||||
waiters: Vec::new(),
|
||||
};
|
||||
assert!(!should_cache_builder(&entry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_builder_records_metrics() {
|
||||
let mut entry = FileBuilderEntry {
|
||||
builder: None,
|
||||
remaining_ranges: 1,
|
||||
waiters: Vec::new(),
|
||||
};
|
||||
let builder = Arc::new(FileRangeBuilder::default());
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
|
||||
assert!(cache_builder_if_needed(
|
||||
&mut entry,
|
||||
&builder,
|
||||
&mut reader_metrics
|
||||
));
|
||||
assert!(entry.builder.is_some());
|
||||
assert_eq!(
|
||||
reader_metrics.metadata_mem_size,
|
||||
builder.memory_size() as isize
|
||||
);
|
||||
assert_eq!(reader_metrics.num_range_builders, 1);
|
||||
|
||||
if entry.builder.take().is_some() {
|
||||
PRUNER_ACTIVE_BUILDERS.dec();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skip_file_range_decrements_and_clears_builder() {
|
||||
let (_env, pruner) = make_test_pruner(1).await;
|
||||
|
||||
// Simulate 3 partition ranges for file 0.
|
||||
let ranges: Vec<PartitionRange> = (0..3).map(|_| file_partition_range(0)).collect();
|
||||
pruner.add_partition_ranges(&ranges);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 3);
|
||||
|
||||
// Manually set a cached builder (simulating a previous cache hit).
|
||||
{
|
||||
let mut entry = pruner.inner.file_entries[0].lock().unwrap();
|
||||
entry.builder = Some(Arc::new(FileRangeBuilder::default()));
|
||||
PRUNER_ACTIVE_BUILDERS.inc();
|
||||
}
|
||||
assert!(pruner.test_has_builder(0));
|
||||
|
||||
// Skip all 3 ranges; the third should clear the builder.
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
for i in 0..3 {
|
||||
let index = RowGroupIndex {
|
||||
index: 0,
|
||||
row_group_index: i as i64,
|
||||
};
|
||||
pruner.skip_file_range(index, &mut reader_metrics);
|
||||
}
|
||||
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 0);
|
||||
assert!(!pruner.test_has_builder(0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_does_not_cache_after_skip_file_range_consumed_all() {
|
||||
let (_env, pruner) = make_test_pruner(1).await;
|
||||
|
||||
// Simulate one range for file 0.
|
||||
let ranges = vec![file_partition_range(0)];
|
||||
pruner.add_partition_ranges(&ranges);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 1);
|
||||
|
||||
// Simulate skip_file_range consuming the last range BEFORE the
|
||||
// background worker finishes. This mirrors the race: a dynamic filter
|
||||
// tightens and manifest-prune fast-skip zeros out remaining_ranges.
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
let index = RowGroupIndex {
|
||||
index: 0,
|
||||
row_group_index: 0,
|
||||
};
|
||||
pruner.skip_file_range(index, &mut reader_metrics);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 0);
|
||||
assert!(!pruner.test_has_builder(0));
|
||||
|
||||
// Now simulate the worker completing: check the caching guard.
|
||||
let entry = pruner.inner.file_entries[0].lock().unwrap();
|
||||
let should_cache = should_cache_builder(&entry);
|
||||
drop(entry);
|
||||
|
||||
assert!(!should_cache);
|
||||
|
||||
// Ensure the gauge was not incremented for a stale builder.
|
||||
// (skip_file_range already decremented it if there was one, but here
|
||||
// there was none, so the gauge should be at baseline.)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_caches_when_ranges_remain() {
|
||||
let (_env, pruner) = make_test_pruner(1).await;
|
||||
|
||||
// Simulate 2 ranges for file 0.
|
||||
let ranges: Vec<PartitionRange> = (0..2).map(|_| file_partition_range(0)).collect();
|
||||
pruner.add_partition_ranges(&ranges);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 2);
|
||||
|
||||
// Consume only 1 range.
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
let index = RowGroupIndex {
|
||||
index: 0,
|
||||
row_group_index: 0,
|
||||
};
|
||||
pruner.skip_file_range(index, &mut reader_metrics);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 1);
|
||||
|
||||
// The worker should still cache because remaining_ranges > 0.
|
||||
let entry = pruner.inner.file_entries[0].lock().unwrap();
|
||||
assert!(should_cache_builder(&entry));
|
||||
}
|
||||
|
||||
// ── Corner case: fast-skip across multiple row groups ─────────────
|
||||
|
||||
/// 1 file × 3 row groups, predicate `ts > 10000ms` prunes the file at
|
||||
/// manifest level. Fast-skipping each of the 3 row groups must return true
|
||||
/// and decrement remaining_ranges to 0.
|
||||
#[tokio::test]
|
||||
async fn try_skip_manifest_pruned_file_range_multi_row_groups() {
|
||||
let predicate_exprs: Vec<Expr> =
|
||||
vec![col("ts").gt(lit(ScalarValue::TimestampMillisecond(Some(10_000), None)))];
|
||||
let (_env, pruner) = make_test_pruner_with_predicate(1, 3, &predicate_exprs).await;
|
||||
|
||||
let ranges = pruner.inner.stream_ctx.partition_ranges();
|
||||
assert_eq!(ranges.len(), 3);
|
||||
pruner.add_partition_ranges(&ranges);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 3);
|
||||
|
||||
let partition_pruner = Arc::new(PartitionPruner::new(pruner.clone(), &ranges));
|
||||
let partition_metrics = make_partition_metrics();
|
||||
|
||||
// Fast-skip each of the 3 row groups.
|
||||
for rg in 0..3 {
|
||||
let index = RowGroupIndex {
|
||||
index: 0, // file_index == 0, no memtables
|
||||
row_group_index: rg,
|
||||
};
|
||||
let skipped =
|
||||
partition_pruner.try_skip_manifest_pruned_file_range(index, &partition_metrics);
|
||||
assert!(skipped, "row group {} should be skipped", rg);
|
||||
}
|
||||
|
||||
// All refs consumed.
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 0);
|
||||
// manifest_pruned_files is CAS'd exactly once (first call).
|
||||
assert!(pruner.test_is_manifest_pruned(0));
|
||||
}
|
||||
|
||||
/// A file whose manifest time range may contain matching rows must not be
|
||||
/// fast-skipped. This protects query correctness over metrics precision.
|
||||
#[tokio::test]
|
||||
async fn try_skip_manifest_pruned_file_range_keeps_overlapping_file() {
|
||||
let predicate_exprs: Vec<Expr> =
|
||||
vec![col("ts").gt(lit(ScalarValue::TimestampMillisecond(Some(500), None)))];
|
||||
let (_env, pruner) = make_test_pruner_with_predicate(1, 2, &predicate_exprs).await;
|
||||
|
||||
let ranges = pruner.inner.stream_ctx.partition_ranges();
|
||||
assert_eq!(ranges.len(), 2);
|
||||
pruner.add_partition_ranges(&ranges);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 2);
|
||||
|
||||
let partition_pruner = Arc::new(PartitionPruner::new(pruner.clone(), &ranges));
|
||||
let partition_metrics = make_partition_metrics();
|
||||
let range_meta = &pruner.inner.stream_ctx.ranges[ranges[0].identifier];
|
||||
let index = range_meta.row_group_indices[0];
|
||||
|
||||
let skipped =
|
||||
partition_pruner.try_skip_manifest_pruned_file_range(index, &partition_metrics);
|
||||
|
||||
assert!(!skipped);
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 2);
|
||||
assert!(!pruner.test_is_manifest_pruned(0));
|
||||
}
|
||||
|
||||
// ── Corner case: add_partition_ranges resets the manifest-pruned flag ──
|
||||
|
||||
#[tokio::test]
|
||||
async fn add_partition_ranges_resets_manifest_pruned_flag() {
|
||||
let predicate_exprs: Vec<Expr> =
|
||||
vec![col("ts").gt(lit(ScalarValue::TimestampMillisecond(Some(10_000), None)))];
|
||||
let (_env, pruner) = make_test_pruner_with_predicate(1, 1, &predicate_exprs).await;
|
||||
|
||||
// Mark file 0 as manifest-pruned.
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
let marked = pruner
|
||||
.inner
|
||||
.try_mark_manifest_pruned(0, &mut reader_metrics);
|
||||
assert!(marked);
|
||||
assert!(pruner.test_is_manifest_pruned(0));
|
||||
assert_eq!(reader_metrics.filter_metrics.files_time_range_pruned, 1);
|
||||
|
||||
// Calling add_partition_ranges must reset the flag.
|
||||
let ranges = vec![file_partition_range(0)];
|
||||
pruner.add_partition_ranges(&ranges);
|
||||
assert!(!pruner.test_is_manifest_pruned(0));
|
||||
// remaining_ranges was also incremented.
|
||||
assert_eq!(pruner.test_remaining_ranges(0), 1);
|
||||
}
|
||||
|
||||
// ── Corner case: prune_file_directly short-circuits via manifest prune ──
|
||||
|
||||
#[tokio::test]
|
||||
async fn prune_file_directly_manifest_pruned_returns_empty_builder() {
|
||||
let predicate_exprs: Vec<Expr> =
|
||||
vec![col("ts").gt(lit(ScalarValue::TimestampMillisecond(Some(10_000), None)))];
|
||||
let (_env, pruner) = make_test_pruner_with_predicate(1, 1, &predicate_exprs).await;
|
||||
|
||||
// Ensure there is no cached builder yet.
|
||||
assert!(!pruner.test_has_builder(0));
|
||||
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
let builder = pruner
|
||||
.prune_file_directly(0, PreFilterMode::SkipFields, &mut reader_metrics)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Should be the default (empty) builder.
|
||||
assert_eq!(
|
||||
builder.memory_size(),
|
||||
FileRangeBuilder::default().memory_size()
|
||||
);
|
||||
// builder must NOT be cached — the manifest-pruned flag is enough.
|
||||
assert!(!pruner.test_has_builder(0));
|
||||
// files_time_range_pruned was recorded.
|
||||
assert_eq!(reader_metrics.filter_metrics.files_time_range_pruned, 1);
|
||||
}
|
||||
|
||||
// ── Corner case: try_mark_manifest_pruned does not double-count ───
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_mark_manifest_pruned_only_counts_first_cas() {
|
||||
let predicate_exprs: Vec<Expr> =
|
||||
vec![col("ts").gt(lit(ScalarValue::TimestampMillisecond(Some(10_000), None)))];
|
||||
let (_env, pruner) = make_test_pruner_with_predicate(1, 1, &predicate_exprs).await;
|
||||
|
||||
// First call: CAS succeeds, metric incremented.
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
let marked = pruner
|
||||
.inner
|
||||
.try_mark_manifest_pruned(0, &mut reader_metrics);
|
||||
assert!(marked);
|
||||
assert_eq!(reader_metrics.filter_metrics.files_time_range_pruned, 1);
|
||||
assert!(pruner.test_is_manifest_pruned(0));
|
||||
|
||||
// Second call: already true, no metric delta.
|
||||
let mut reader_metrics2 = ReaderMetrics::default();
|
||||
let marked2 = pruner
|
||||
.inner
|
||||
.try_mark_manifest_pruned(0, &mut reader_metrics2);
|
||||
assert!(marked2);
|
||||
assert_eq!(reader_metrics2.filter_metrics.files_time_range_pruned, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,11 +28,14 @@ use common_telemetry::tracing::Instrument;
|
||||
use common_telemetry::{debug, error, tracing, warn};
|
||||
use common_time::range::TimestampRange;
|
||||
use datafusion::physical_plan::expressions::DynamicFilterPhysicalExpr;
|
||||
use datafusion_common::Column;
|
||||
use datafusion_common::pruning::PruningStatistics;
|
||||
use datafusion_common::{Column, ScalarValue};
|
||||
use datafusion_expr::Expr;
|
||||
use datafusion_expr::utils::expr_to_columns;
|
||||
use datatypes::arrow::array::{ArrayRef, BooleanArray, UInt64Array};
|
||||
use datatypes::extension::json::is_structured_json_field;
|
||||
use datatypes::types::json_type::JsonNativeType;
|
||||
use datatypes::value::timestamp_to_scalar_value;
|
||||
use futures::StreamExt;
|
||||
use itertools::Itertools;
|
||||
use partition::expr::PartitionExpr;
|
||||
@@ -1053,7 +1056,7 @@ impl ScanInput {
|
||||
ranges
|
||||
}
|
||||
|
||||
fn predicate_for_file(&self, file: &FileHandle) -> Option<Predicate> {
|
||||
pub(crate) fn predicate_for_file(&self, file: &FileHandle) -> Option<Predicate> {
|
||||
if self.should_skip_region_partition(file) {
|
||||
self.predicate.predicate_without_region().cloned()
|
||||
} else {
|
||||
@@ -1071,7 +1074,56 @@ impl ScanInput {
|
||||
}
|
||||
}
|
||||
|
||||
/// Tries to build file-level pruning statistics using only the [FileHandle]'s manifest-level
|
||||
/// time range, without reading any parquet metadata.
|
||||
///
|
||||
/// Returns `None` if timestamp unit conversion overflows (conservative: keep the file).
|
||||
fn try_file_level_pruning_stats(&self, file: &FileHandle) -> Option<FileLevelPruningStats> {
|
||||
let (ts_min, ts_max) = file.time_range();
|
||||
let time_index = self.mapper.metadata().time_index_column();
|
||||
let time_index_unit = time_index.column_schema.data_type.as_timestamp()?.unit();
|
||||
|
||||
// Convert file timestamps to the time index column's unit. Use `convert_to_ceil` for
|
||||
// the upper bound to avoid accidentally shrinking the manifest range.
|
||||
let min_ts = ts_min.convert_to(time_index_unit)?;
|
||||
let max_ts = ts_max.convert_to_ceil(time_index_unit)?;
|
||||
|
||||
Some(FileLevelPruningStats {
|
||||
min_scalar: timestamp_to_scalar_value(time_index_unit, Some(min_ts.value())),
|
||||
max_scalar: timestamp_to_scalar_value(time_index_unit, Some(max_ts.value())),
|
||||
time_index_col_name: time_index.column_schema.name.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Checks whether a file can be definitively pruned using only its manifest-level
|
||||
/// time range and the current predicate, without reading any parquet metadata.
|
||||
///
|
||||
/// Returns `true` if [PruningStatistics] proves the file cannot contain matching rows.
|
||||
#[inline]
|
||||
pub(crate) fn can_manifest_prune_file(&self, file: &FileHandle) -> bool {
|
||||
let predicate = self.predicate_for_file(file);
|
||||
self.manifest_prunes_file(file, predicate.as_ref())
|
||||
}
|
||||
|
||||
fn manifest_prunes_file(&self, file: &FileHandle, predicate: Option<&Predicate>) -> bool {
|
||||
if let Some(pred) = predicate
|
||||
&& !pred.is_empty()
|
||||
&& let Some(file_level_stats) = self.try_file_level_pruning_stats(file)
|
||||
{
|
||||
let pruning_results = pred.prune_with_stats(
|
||||
&file_level_stats,
|
||||
self.mapper.metadata().schema.arrow_schema(),
|
||||
);
|
||||
pruning_results.first() == Some(&false)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Prunes a file to scan and returns the builder to build readers.
|
||||
///
|
||||
/// This is the public entry point used by direct tests and non-pruner callers.
|
||||
/// It performs its own manifest-level pruning check internally.
|
||||
#[tracing::instrument(
|
||||
skip_all,
|
||||
fields(
|
||||
@@ -1086,6 +1138,31 @@ impl ScanInput {
|
||||
reader_metrics: &mut ReaderMetrics,
|
||||
) -> Result<FileRangeBuilder> {
|
||||
let predicate = self.predicate_for_file(file);
|
||||
|
||||
// Early file-level pruning using manifest time range before any parquet metadata access.
|
||||
if self.manifest_prunes_file(file, predicate.as_ref()) {
|
||||
reader_metrics.filter_metrics.files_time_range_pruned += 1;
|
||||
return Ok(FileRangeBuilder::default());
|
||||
}
|
||||
|
||||
self.prune_file_after_manifest_check(file, pre_filter_mode, predicate, reader_metrics)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Second half of `prune_file` — performs the actual parquet metadata /
|
||||
/// reader setup. Callers that already performed manifest-level pruning
|
||||
/// (e.g. the `Pruner` via its shared `manifest_pruned_files` cache) should
|
||||
/// call this directly to avoid a redundant manifest check.
|
||||
///
|
||||
/// `predicate` is the result of `self.predicate_for_file(file)` computed
|
||||
/// externally so the caller can reuse it if needed.
|
||||
pub(crate) async fn prune_file_after_manifest_check(
|
||||
&self,
|
||||
file: &FileHandle,
|
||||
pre_filter_mode: PreFilterMode,
|
||||
predicate: Option<Predicate>,
|
||||
reader_metrics: &mut ReaderMetrics,
|
||||
) -> Result<FileRangeBuilder> {
|
||||
let may_build_selective_row_selection = predicate.is_some();
|
||||
let decode_pk_values = !self.compaction
|
||||
&& self
|
||||
@@ -1305,6 +1382,57 @@ impl ScanInput {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lightweight [PruningStatistics] that only uses the file-level time range from manifest
|
||||
/// metadata, avoiding any parquet metadata reads. Used for early file-level pruning before
|
||||
/// accessing row-group-level statistics.
|
||||
pub(crate) struct FileLevelPruningStats {
|
||||
/// Scalar value for the file's minimum timestamp in the time index column's unit.
|
||||
pub(crate) min_scalar: ScalarValue,
|
||||
/// Scalar value for the file's maximum timestamp in the time index column's unit.
|
||||
pub(crate) max_scalar: ScalarValue,
|
||||
/// Name of the time index column.
|
||||
pub(crate) time_index_col_name: String,
|
||||
}
|
||||
|
||||
impl PruningStatistics for FileLevelPruningStats {
|
||||
fn min_values(&self, column: &Column) -> Option<ArrayRef> {
|
||||
if column.name == self.time_index_col_name {
|
||||
ScalarValue::iter_to_array(std::iter::once(self.min_scalar.clone())).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn max_values(&self, column: &Column) -> Option<ArrayRef> {
|
||||
if column.name == self.time_index_col_name {
|
||||
ScalarValue::iter_to_array(std::iter::once(self.max_scalar.clone())).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn num_containers(&self) -> usize {
|
||||
1
|
||||
}
|
||||
|
||||
fn null_counts(&self, column: &Column) -> Option<ArrayRef> {
|
||||
if column.name == self.time_index_col_name {
|
||||
// The time index column is NOT NULL.
|
||||
Some(Arc::new(UInt64Array::from(vec![0u64])))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn row_counts(&self, _column: &Column) -> Option<ArrayRef> {
|
||||
None
|
||||
}
|
||||
|
||||
fn contained(&self, _column: &Column, _values: &HashSet<ScalarValue>) -> Option<BooleanArray> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl ScanInput {
|
||||
/// Returns SST file ids to scan.
|
||||
@@ -1861,9 +1989,15 @@ impl PredicateGroup {
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::physical_plan::expressions::lit as physical_lit;
|
||||
use common_time::timestamp::{TimeUnit, Timestamp};
|
||||
use datafusion::physical_plan::expressions::{
|
||||
binary as physical_binary, col as physical_col, lit as physical_lit,
|
||||
};
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_expr::{col, lit};
|
||||
use datafusion_expr::{Operator, col, lit};
|
||||
use datatypes::arrow::datatypes::{
|
||||
DataType as ArrowDataType, Field, Schema as ArrowSchema, TimeUnit as ArrowTimeUnit,
|
||||
};
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::ColumnSchema;
|
||||
use datatypes::types::json_type::JsonObjectType;
|
||||
@@ -1877,6 +2011,7 @@ mod tests {
|
||||
use crate::error::InvalidMetadataSnafu;
|
||||
use crate::read::range_cache::ScanRequestFingerprintBuilder;
|
||||
use crate::read::read_columns::ReadColumn;
|
||||
use crate::sst::file::FileMeta;
|
||||
use crate::test_util::memtable_util::metadata_with_primary_key;
|
||||
use crate::test_util::scheduler_util::SchedulerEnv;
|
||||
|
||||
@@ -1904,6 +2039,60 @@ mod tests {
|
||||
lit(ScalarValue::TimestampMillisecond(Some(val), None))
|
||||
}
|
||||
|
||||
fn metadata_with_time_index_unit(unit: TimeUnit) -> RegionMetadataRef {
|
||||
let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
|
||||
builder
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"k0".to_string(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
false,
|
||||
),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: 0,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"k1".to_string(),
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
false,
|
||||
),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: 1,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"ts".to_string(),
|
||||
ConcreteDataType::timestamp_datatype(unit),
|
||||
false,
|
||||
),
|
||||
semantic_type: SemanticType::Timestamp,
|
||||
column_id: 2,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"v0".to_string(),
|
||||
ConcreteDataType::int64_datatype(),
|
||||
true,
|
||||
),
|
||||
semantic_type: SemanticType::Field,
|
||||
column_id: 3,
|
||||
})
|
||||
.primary_key(vec![0, 1]);
|
||||
|
||||
Arc::new(builder.build().unwrap())
|
||||
}
|
||||
|
||||
fn file_handle_with_time_range(start: Timestamp, end: Timestamp) -> FileHandle {
|
||||
FileHandle::new(
|
||||
FileMeta {
|
||||
time_range: (start, end),
|
||||
..Default::default()
|
||||
},
|
||||
Arc::new(crate::sst::file_purger::NoopFilePurger),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fill_json_nested_paths_from_hint() -> Result<()> {
|
||||
fn json_projection_test_metadata() -> Result<RegionMetadataRef> {
|
||||
@@ -2129,6 +2318,177 @@ mod tests {
|
||||
assert_eq!(1, predicate_without_region.dyn_filters().len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_level_pruning_stats_prunes_old_file() {
|
||||
let ts_col_name = "ts";
|
||||
let predicate = Predicate::new(vec![col(ts_col_name).gt(ts_lit(1000))]);
|
||||
let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
|
||||
ts_col_name,
|
||||
ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None),
|
||||
false,
|
||||
)]));
|
||||
|
||||
// File with time range [0ms, 500ms] is completely before `ts > 1000ms`.
|
||||
let stats = FileLevelPruningStats {
|
||||
min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
|
||||
max_scalar: ScalarValue::TimestampMillisecond(Some(500), None),
|
||||
time_index_col_name: ts_col_name.to_string(),
|
||||
};
|
||||
assert_eq!(
|
||||
vec![false],
|
||||
predicate.prune_with_stats(&stats, &arrow_schema)
|
||||
);
|
||||
|
||||
// File with time range [0ms, 2000ms] overlaps `ts > 1000ms`, so keep it.
|
||||
let stats = FileLevelPruningStats {
|
||||
min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
|
||||
max_scalar: ScalarValue::TimestampMillisecond(Some(2000), None),
|
||||
time_index_col_name: ts_col_name.to_string(),
|
||||
};
|
||||
assert_eq!(
|
||||
vec![true],
|
||||
predicate.prune_with_stats(&stats, &arrow_schema)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_level_pruning_stats_no_predicate_keeps_all() {
|
||||
let predicate = Predicate::new(vec![]);
|
||||
assert!(predicate.is_empty());
|
||||
|
||||
let stats = FileLevelPruningStats {
|
||||
min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
|
||||
max_scalar: ScalarValue::TimestampMillisecond(Some(500), None),
|
||||
time_index_col_name: "ts".to_string(),
|
||||
};
|
||||
let arrow_schema = Arc::new(ArrowSchema::new(Vec::<Field>::new()));
|
||||
assert_eq!(
|
||||
vec![true],
|
||||
predicate.prune_with_stats(&stats, &arrow_schema)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_level_pruning_stats_ceil_max_unit_conversion() {
|
||||
let metadata = metadata_with_time_index_unit(TimeUnit::Millisecond);
|
||||
let input = new_scan_input(metadata, vec![]).await;
|
||||
let file = file_handle_with_time_range(
|
||||
Timestamp::new(1_000_001, TimeUnit::Nanosecond),
|
||||
Timestamp::new(1_000_001, TimeUnit::Nanosecond),
|
||||
);
|
||||
|
||||
let stats = input.try_file_level_pruning_stats(&file).unwrap();
|
||||
assert_eq!(
|
||||
ScalarValue::TimestampMillisecond(Some(1), None),
|
||||
stats.min_scalar
|
||||
);
|
||||
assert_eq!(
|
||||
ScalarValue::TimestampMillisecond(Some(2), None),
|
||||
stats.max_scalar
|
||||
);
|
||||
|
||||
// The actual max timestamp is slightly greater than 1ms. It must be kept for `ts > 1ms`.
|
||||
let predicate = Predicate::new(vec![col("ts").gt(ts_lit(1))]);
|
||||
assert_eq!(
|
||||
vec![true],
|
||||
predicate.prune_with_stats(&stats, input.mapper.metadata().schema.arrow_schema())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_level_pruning_stats_overflow_keeps_file() {
|
||||
let metadata = metadata_with_time_index_unit(TimeUnit::Nanosecond);
|
||||
let input = new_scan_input(metadata, vec![]).await;
|
||||
let file = file_handle_with_time_range(
|
||||
Timestamp::new(0, TimeUnit::Second),
|
||||
Timestamp::new(i64::MAX, TimeUnit::Second),
|
||||
);
|
||||
|
||||
assert!(input.try_file_level_pruning_stats(&file).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_level_pruning_stats_keeps_inclusive_boundary() {
|
||||
let ts_col_name = "ts";
|
||||
let predicate = Predicate::new(vec![col(ts_col_name).gt_eq(ts_lit(1000))]);
|
||||
let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
|
||||
ts_col_name,
|
||||
ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None),
|
||||
false,
|
||||
)]));
|
||||
let stats = FileLevelPruningStats {
|
||||
min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
|
||||
max_scalar: ScalarValue::TimestampMillisecond(Some(1000), None),
|
||||
time_index_col_name: ts_col_name.to_string(),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
vec![true],
|
||||
predicate.prune_with_stats(&stats, &arrow_schema)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_level_pruning_with_dyn_filter_only_predicate() {
|
||||
let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
|
||||
let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
|
||||
let predicate_group = PredicateGroup::new(metadata.as_ref(), &[]).unwrap();
|
||||
predicate_group.add_dyn_filters(vec![Arc::new(DynamicFilterPhysicalExpr::new(
|
||||
vec![],
|
||||
physical_lit(false),
|
||||
))]);
|
||||
let input = ScanInput::new(SchedulerEnv::new().await.access_layer.clone(), mapper)
|
||||
.with_predicate(predicate_group);
|
||||
let file = file_handle_with_time_range(
|
||||
Timestamp::new_millisecond(0),
|
||||
Timestamp::new_millisecond(1000),
|
||||
);
|
||||
let mut reader_metrics = ReaderMetrics::default();
|
||||
|
||||
let builder = input
|
||||
.prune_file(&file, PreFilterMode::SkipFields, &mut reader_metrics)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(1, reader_metrics.filter_metrics.files_time_range_pruned);
|
||||
let mut ranges = SmallVec::new();
|
||||
builder.build_ranges(-1, &mut ranges);
|
||||
assert!(ranges.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_manifest_pruning_observes_dynamic_filter_update() {
|
||||
let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
|
||||
let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
|
||||
let predicate_group = PredicateGroup::new(metadata.as_ref(), &[]).unwrap();
|
||||
let arrow_schema = metadata.schema.arrow_schema();
|
||||
let ts_expr = physical_col("ts", arrow_schema.as_ref()).unwrap();
|
||||
let dyn_filter = Arc::new(DynamicFilterPhysicalExpr::new(
|
||||
vec![ts_expr.clone()],
|
||||
physical_lit(true),
|
||||
));
|
||||
predicate_group.add_dyn_filters(vec![dyn_filter.clone()]);
|
||||
let input = ScanInput::new(SchedulerEnv::new().await.access_layer.clone(), mapper)
|
||||
.with_predicate(predicate_group);
|
||||
let file = file_handle_with_time_range(
|
||||
Timestamp::new_millisecond(0),
|
||||
Timestamp::new_millisecond(1000),
|
||||
);
|
||||
|
||||
assert!(!input.can_manifest_prune_file(&file));
|
||||
|
||||
let updated = physical_binary(
|
||||
ts_expr,
|
||||
Operator::Gt,
|
||||
physical_lit(ScalarValue::TimestampMillisecond(Some(1000), None)),
|
||||
arrow_schema.as_ref(),
|
||||
)
|
||||
.unwrap();
|
||||
dyn_filter.update(updated).unwrap();
|
||||
|
||||
assert!(input.can_manifest_prune_file(&file));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_range_pre_filter_mode() {
|
||||
let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
|
||||
|
||||
@@ -195,6 +195,8 @@ pub(crate) struct ScanMetricsSet {
|
||||
pruner_cache_miss: usize,
|
||||
/// Duration spent waiting for pruner to build file ranges.
|
||||
pruner_prune_cost: Duration,
|
||||
/// Number of files filtered by manifest time-range pruning.
|
||||
files_time_range_pruned: usize,
|
||||
/// Number of record batches read from SST.
|
||||
num_sst_record_batches: usize,
|
||||
/// Number of batches decoded from SST.
|
||||
@@ -326,6 +328,7 @@ impl fmt::Debug for ScanMetricsSet {
|
||||
pruner_cache_hit,
|
||||
pruner_cache_miss,
|
||||
pruner_prune_cost,
|
||||
files_time_range_pruned,
|
||||
num_sst_record_batches,
|
||||
num_sst_batches,
|
||||
num_sst_rows,
|
||||
@@ -390,6 +393,9 @@ impl fmt::Debug for ScanMetricsSet {
|
||||
}
|
||||
|
||||
// Write non-zero filter counters
|
||||
if *files_time_range_pruned > 0 {
|
||||
write!(f, ", \"files_time_range_pruned\":{files_time_range_pruned}")?;
|
||||
}
|
||||
if *rg_fulltext_filtered > 0 {
|
||||
write!(f, ", \"rg_fulltext_filtered\":{rg_fulltext_filtered}")?;
|
||||
}
|
||||
@@ -689,6 +695,7 @@ impl ScanMetricsSet {
|
||||
pruner_cache_hit,
|
||||
pruner_cache_miss,
|
||||
pruner_prune_cost,
|
||||
files_time_range_pruned,
|
||||
inverted_index_apply_metrics,
|
||||
bloom_filter_apply_metrics,
|
||||
fulltext_index_apply_metrics,
|
||||
@@ -706,6 +713,8 @@ impl ScanMetricsSet {
|
||||
self.build_parts_cost += *build_cost;
|
||||
self.sst_scan_cost += *scan_cost;
|
||||
|
||||
self.files_time_range_pruned += *files_time_range_pruned;
|
||||
|
||||
self.rg_total += *rg_total;
|
||||
self.rg_fulltext_filtered += *rg_fulltext_filtered;
|
||||
self.rg_inverted_filtered += *rg_inverted_filtered;
|
||||
|
||||
@@ -661,6 +661,13 @@ pub(crate) async fn build_flat_sources(
|
||||
);
|
||||
ordered_sources[position] = Some(Box::pin(stream) as _);
|
||||
} else if stream_ctx.is_file_range_index(*index) {
|
||||
// Common manifest-level fast-skip shared by SeqScan and UnorderedScan.
|
||||
// Compaction should keep reading its selected input ranges completely.
|
||||
if !compaction
|
||||
&& partition_pruner.try_skip_manifest_pruned_file_range(*index, part_metrics)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if let Some(semaphore_ref) = semaphore.as_ref() {
|
||||
// run in parallel, controlled by semaphore
|
||||
let stream_ctx = stream_ctx.clone();
|
||||
|
||||
@@ -133,6 +133,12 @@ impl UnorderedScan {
|
||||
yield record_batch?;
|
||||
}
|
||||
} else if stream_ctx.is_file_range_index(*index) {
|
||||
// Common manifest-level fast-skip shared by UnorderedScan and SeqScan.
|
||||
if partition_pruner
|
||||
.try_skip_manifest_pruned_file_range(*index, &part_metrics)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let stream = scan_flat_file_ranges(
|
||||
stream_ctx.clone(),
|
||||
part_metrics.clone(),
|
||||
|
||||
@@ -1428,6 +1428,8 @@ pub(crate) struct ReaderFilterMetrics {
|
||||
pub(crate) pruner_cache_miss: usize,
|
||||
/// Duration spent waiting for pruner to build file ranges.
|
||||
pub(crate) pruner_prune_cost: Duration,
|
||||
/// Number of files filtered by manifest time-range pruning.
|
||||
pub(crate) files_time_range_pruned: usize,
|
||||
}
|
||||
|
||||
impl ReaderFilterMetrics {
|
||||
@@ -1460,6 +1462,7 @@ impl ReaderFilterMetrics {
|
||||
self.pruner_cache_hit += other.pruner_cache_hit;
|
||||
self.pruner_cache_miss += other.pruner_cache_miss;
|
||||
self.pruner_prune_cost += other.pruner_prune_cost;
|
||||
self.files_time_range_pruned += other.files_time_range_pruned;
|
||||
|
||||
// Merge optional applier metrics
|
||||
if let Some(other_metrics) = &other.inverted_index_apply_metrics {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use api::v1::CommentOnExpr;
|
||||
use common_error::ext::BoxedError;
|
||||
use common_meta::cache_invalidator::Context;
|
||||
use common_meta::procedure_executor::ExecutorContext;
|
||||
use common_meta::rpc::ddl::{CommentObjectType, CommentOnTask, DdlTask, SubmitDdlTaskRequest};
|
||||
use common_query::Output;
|
||||
@@ -23,7 +24,9 @@ use snafu::ResultExt;
|
||||
use sql::ast::ObjectNamePartExt;
|
||||
use sql::statements::comment::{Comment, CommentObject};
|
||||
|
||||
use crate::error::{ExecuteDdlSnafu, ExternalSnafu, InvalidSqlSnafu, Result};
|
||||
use crate::error::{
|
||||
self, ExecuteDdlSnafu, ExternalSnafu, InvalidSqlSnafu, Result, TableMetadataManagerSnafu,
|
||||
};
|
||||
use crate::statement::StatementExecutor;
|
||||
use crate::utils::to_meta_query_context;
|
||||
|
||||
@@ -39,7 +42,15 @@ impl StatementExecutor {
|
||||
///
|
||||
/// A `Result` containing the `Output` of the operation, or an error if the operation fails.
|
||||
pub async fn comment(&self, stmt: Comment, query_ctx: QueryContextRef) -> Result<Output> {
|
||||
let comment_on_task = self.create_comment_on_task_from_stmt(stmt, &query_ctx)?;
|
||||
let mut comment_on_task = self.create_comment_on_task_from_stmt(stmt, &query_ctx)?;
|
||||
comment_on_task
|
||||
.enrich_object_id(
|
||||
self.table_metadata_manager.table_name_manager(),
|
||||
self.flow_metadata_manager.flow_name_manager(),
|
||||
)
|
||||
.await
|
||||
.context(TableMetadataManagerSnafu)?;
|
||||
let cache_idents = comment_on_task.cache_idents();
|
||||
|
||||
let request = SubmitDdlTaskRequest::new(
|
||||
to_meta_query_context(query_ctx),
|
||||
@@ -49,8 +60,15 @@ impl StatementExecutor {
|
||||
self.procedure_executor
|
||||
.submit_ddl_task(&ExecutorContext::default(), request)
|
||||
.await
|
||||
.context(ExecuteDdlSnafu)
|
||||
.map(|_| Output::new_with_affected_rows(0))
|
||||
.context(ExecuteDdlSnafu)?;
|
||||
|
||||
// Invalidates local cache ASAP.
|
||||
self.cache_invalidator
|
||||
.invalidate(&Context::default(), &cache_idents)
|
||||
.await
|
||||
.context(error::InvalidateTableCacheSnafu)?;
|
||||
|
||||
Ok(Output::new_with_affected_rows(0))
|
||||
}
|
||||
|
||||
pub async fn comment_by_expr(
|
||||
@@ -58,7 +76,15 @@ impl StatementExecutor {
|
||||
expr: CommentOnExpr,
|
||||
query_ctx: QueryContextRef,
|
||||
) -> Result<Output> {
|
||||
let comment_on_task = self.create_comment_on_task_from_expr(expr)?;
|
||||
let mut comment_on_task = self.create_comment_on_task_from_expr(expr)?;
|
||||
comment_on_task
|
||||
.enrich_object_id(
|
||||
self.table_metadata_manager.table_name_manager(),
|
||||
self.flow_metadata_manager.flow_name_manager(),
|
||||
)
|
||||
.await
|
||||
.context(TableMetadataManagerSnafu)?;
|
||||
let cache_idents = comment_on_task.cache_idents();
|
||||
|
||||
let request = SubmitDdlTaskRequest::new(
|
||||
to_meta_query_context(query_ctx),
|
||||
@@ -68,8 +94,15 @@ impl StatementExecutor {
|
||||
self.procedure_executor
|
||||
.submit_ddl_task(&ExecutorContext::default(), request)
|
||||
.await
|
||||
.context(ExecuteDdlSnafu)
|
||||
.map(|_| Output::new_with_affected_rows(0))
|
||||
.context(ExecuteDdlSnafu)?;
|
||||
|
||||
// Invalidates local cache ASAP.
|
||||
self.cache_invalidator
|
||||
.invalidate(&Context::default(), &cache_idents)
|
||||
.await
|
||||
.context(error::InvalidateTableCacheSnafu)?;
|
||||
|
||||
Ok(Output::new_with_affected_rows(0))
|
||||
}
|
||||
|
||||
fn create_comment_on_task_from_expr(&self, expr: CommentOnExpr) -> Result<CommentOnTask> {
|
||||
|
||||
@@ -26,6 +26,16 @@ use datafusion_expr::expr::{Exists, InSubquery};
|
||||
use datafusion_expr::utils::expr_to_columns;
|
||||
use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, Subquery, col as col_fn};
|
||||
use datafusion_optimizer::analyzer::AnalyzerRule;
|
||||
use datafusion_optimizer::decorrelate_lateral_join::DecorrelateLateralJoin;
|
||||
use datafusion_optimizer::decorrelate_predicate_subquery::DecorrelatePredicateSubquery;
|
||||
use datafusion_optimizer::eliminate_filter::EliminateFilter;
|
||||
use datafusion_optimizer::extract_equijoin_predicate::ExtractEquijoinPredicate;
|
||||
use datafusion_optimizer::filter_null_join_keys::FilterNullJoinKeys;
|
||||
use datafusion_optimizer::optimizer::Optimizer;
|
||||
use datafusion_optimizer::propagate_empty_relation::PropagateEmptyRelation;
|
||||
use datafusion_optimizer::push_down_filter::PushDownFilter;
|
||||
use datafusion_optimizer::rewrite_set_comparison::RewriteSetComparison;
|
||||
use datafusion_optimizer::scalar_subquery_to_join::ScalarSubqueryToJoin;
|
||||
use promql::extension_plan::SeriesDivide;
|
||||
use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
|
||||
use table::metadata::TableType;
|
||||
@@ -111,6 +121,8 @@ impl AnalyzerRule for DistPlannerAnalyzer {
|
||||
// Aligned with the behavior in `datafusion_optimizer::OptimizerContext::new()`.
|
||||
config.optimizer.filter_null_join_keys = true;
|
||||
let config = Arc::new(config);
|
||||
let opt = config.extensions.get::<DistPlannerOptions>();
|
||||
let allow_fallback = opt.map(|o| o.allow_query_fallback).unwrap_or(false);
|
||||
|
||||
// When the query is running under a scheduled Flow context, carry the
|
||||
// logical "now" so that `SimplifyExpressions` does not constant-fold
|
||||
@@ -129,9 +141,31 @@ impl AnalyzerRule for DistPlannerAnalyzer {
|
||||
let plan = plan
|
||||
.rewrite_with_subqueries(&mut PlanTreeExpressionSimplifier::new(optimizer_context))?
|
||||
.data;
|
||||
let fallback_plan = plan.clone();
|
||||
|
||||
let opt = config.extensions.get::<DistPlannerOptions>();
|
||||
let allow_fallback = opt.map(|o| o.allow_query_fallback).unwrap_or(false);
|
||||
// Run a filter-focused optimizer subset before MergeScan wraps remote
|
||||
// inputs. MergeScan intentionally hides its remote_input from later
|
||||
// optimizer passes; this pass only normalizes/decorrelates enough for
|
||||
// DataFusion's PushDownFilter to put side-local predicates into scans.
|
||||
// Keep this narrow: rules like PushDownLimit, OptimizeProjections, and
|
||||
// DISTINCT rewrites can change global distributed-planning boundaries.
|
||||
let optimizer_context = PatchOptimizerContext {
|
||||
inner: datafusion_optimizer::OptimizerContext::new(),
|
||||
config: config.clone(),
|
||||
scheduled_time,
|
||||
};
|
||||
let plan = match pre_merge_scan_optimizer().optimize(plan, &optimizer_context, |_, _| {}) {
|
||||
Ok(plan) => plan,
|
||||
Err(err) => {
|
||||
if allow_fallback {
|
||||
common_telemetry::warn!(err; "Failed to pre-optimize plan, using fallback plan rewriter for plan: {fallback_plan}");
|
||||
PUSH_DOWN_FALLBACK_ERRORS_TOTAL.inc();
|
||||
return self.use_fallback(fallback_plan);
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let result = match self.try_push_down(plan.clone()) {
|
||||
Ok(plan) => plan,
|
||||
@@ -140,7 +174,7 @@ impl AnalyzerRule for DistPlannerAnalyzer {
|
||||
common_telemetry::warn!(err; "Failed to push down plan, using fallback plan rewriter for plan: {plan}");
|
||||
// if push down failed, use fallback plan rewriter
|
||||
PUSH_DOWN_FALLBACK_ERRORS_TOTAL.inc();
|
||||
self.use_fallback(plan)?
|
||||
self.use_fallback(fallback_plan)?
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -151,6 +185,43 @@ impl AnalyzerRule for DistPlannerAnalyzer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the small optimizer pre-pass that runs before `MergeScan` wrapping.
|
||||
///
|
||||
/// This is intentionally not DataFusion's full optimizer. After
|
||||
/// `PlanRewriter` wraps remote table scans in `MergeScan`,
|
||||
/// `MergeScanLogicalPlan::inputs()` hides `remote_input`, so ordinary optimizer
|
||||
/// rules can no longer see into the remote side. The main rule we need here is
|
||||
/// `PushDownFilter`: it moves side-local join/filter predicates into
|
||||
/// `TableScan.filters`, where region pruning and scan-level pruning can use
|
||||
/// them.
|
||||
///
|
||||
/// The rules before `PushDownFilter` are only the minimum cleanup/rewrite steps
|
||||
/// needed to make that filter pushdown safe around subqueries and set
|
||||
/// comparisons. For example, `RewriteSetComparison` handles ANY/ALL before they
|
||||
/// can become scan filters, and the decorrelation/subquery rules expose
|
||||
/// supported predicates as joins/filters instead of leaving raw subquery
|
||||
/// expressions under a scan.
|
||||
///
|
||||
/// Keep this list narrow. Do not add broad plan-shaping rules such as
|
||||
/// `PushDownLimit`, projection optimization, DISTINCT rewrites, or join-type
|
||||
/// rewrites here: those can change the local/remote distributed boundary or
|
||||
/// degrade unrelated planning diagnostics. Such rules belong either before this
|
||||
/// analyzer or after distributed planning, not in this pre-MergeScan,
|
||||
/// filter-focused pass.
|
||||
fn pre_merge_scan_optimizer() -> Optimizer {
|
||||
Optimizer::with_rules(vec![
|
||||
Arc::new(RewriteSetComparison::new()),
|
||||
Arc::new(DecorrelatePredicateSubquery::new()),
|
||||
Arc::new(ScalarSubqueryToJoin::new()),
|
||||
Arc::new(DecorrelateLateralJoin::new()),
|
||||
Arc::new(ExtractEquijoinPredicate::new()),
|
||||
Arc::new(EliminateFilter::new()),
|
||||
Arc::new(PropagateEmptyRelation::new()),
|
||||
Arc::new(FilterNullJoinKeys::default()),
|
||||
Arc::new(PushDownFilter::new()),
|
||||
])
|
||||
}
|
||||
|
||||
impl DistPlannerAnalyzer {
|
||||
/// Try push down as many nodes as possible
|
||||
fn try_push_down(&self, plan: LogicalPlan) -> DfResult<LogicalPlan> {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeSet, HashSet};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -29,7 +29,8 @@ use datafusion::execution::SessionState;
|
||||
use datafusion::functions_aggregate::expr_fn::avg;
|
||||
use datafusion::functions_aggregate::min_max::{max, min};
|
||||
use datafusion::prelude::SessionContext;
|
||||
use datafusion_common::{JoinType, ScalarValue};
|
||||
use datafusion_common::tree_node::TreeNodeRecursion;
|
||||
use datafusion_common::{ExprSchema, JoinType, ScalarValue};
|
||||
use datafusion_expr::expr::{Exists, ScalarFunction};
|
||||
use datafusion_expr::{
|
||||
AggregateUDF, Expr, ExprSchemable as _, LogicalPlanBuilder, Operator, Subquery, binary_expr,
|
||||
@@ -99,14 +100,92 @@ fn collect_merge_scan_remote_dyn_filter_producer_id_list(
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn assert_remote_table_scan_filters_are_safe(plan: &LogicalPlan) {
|
||||
let mut checked_filters = 0;
|
||||
assert_remote_table_scan_filters_are_safe_inner(plan, false, &mut checked_filters);
|
||||
assert!(
|
||||
checked_filters > 0,
|
||||
"expected at least one remote TableScan filter in plan:\n{plan}"
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_remote_table_scan_filters_are_safe_inner(
|
||||
plan: &LogicalPlan,
|
||||
in_merge_scan_remote_input: bool,
|
||||
checked_filters: &mut usize,
|
||||
) {
|
||||
if let LogicalPlan::Extension(extension) = plan
|
||||
&& let Some(merge_scan) = extension
|
||||
.node
|
||||
.as_any()
|
||||
.downcast_ref::<MergeScanLogicalPlan>()
|
||||
{
|
||||
assert_remote_table_scan_filters_are_safe_inner(merge_scan.input(), true, checked_filters);
|
||||
}
|
||||
|
||||
if in_merge_scan_remote_input && let LogicalPlan::TableScan(table_scan) = plan {
|
||||
for filter in &table_scan.filters {
|
||||
assert_table_scan_filter_is_remote_safe(table_scan, filter);
|
||||
*checked_filters += 1;
|
||||
}
|
||||
}
|
||||
|
||||
for child in plan.inputs() {
|
||||
assert_remote_table_scan_filters_are_safe_inner(
|
||||
child,
|
||||
in_merge_scan_remote_input,
|
||||
checked_filters,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_table_scan_filter_is_remote_safe(
|
||||
table_scan: &datafusion_expr::logical_plan::TableScan,
|
||||
filter: &Expr,
|
||||
) {
|
||||
filter
|
||||
.apply(|expr| match expr {
|
||||
Expr::Exists(_)
|
||||
| Expr::InSubquery(_)
|
||||
| Expr::ScalarSubquery(_)
|
||||
| Expr::SetComparison(_)
|
||||
| Expr::OuterReferenceColumn(_, _) => {
|
||||
panic!("remote TableScan filter contains non-scan-local expression: {filter}")
|
||||
}
|
||||
_ => Ok(TreeNodeRecursion::Continue),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let mut columns = HashSet::new();
|
||||
expr_to_columns(filter, &mut columns).unwrap();
|
||||
for column in columns {
|
||||
assert!(
|
||||
table_scan
|
||||
.projected_schema
|
||||
.field_from_column(&column)
|
||||
.is_ok(),
|
||||
"remote TableScan filter references non-scan column {column}: {filter}\nscan schema: {:?}",
|
||||
table_scan.projected_schema
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct TestTable;
|
||||
|
||||
impl TestTable {
|
||||
pub fn table_with_name(table_id: TableId, name: String) -> TableRef {
|
||||
Self::table_with_filter_pushdown(table_id, name, FilterPushDownType::Unsupported)
|
||||
}
|
||||
|
||||
pub fn table_with_filter_pushdown(
|
||||
table_id: TableId,
|
||||
name: String,
|
||||
filter_pushdown: FilterPushDownType,
|
||||
) -> TableRef {
|
||||
let data_source = Arc::new(TestDataSource::new(Self::schema()));
|
||||
let table = Table::new(
|
||||
Self::table_info(table_id, name, "test_engine".to_string()),
|
||||
FilterPushDownType::Unsupported,
|
||||
filter_pushdown,
|
||||
data_source,
|
||||
);
|
||||
Arc::new(table)
|
||||
@@ -1460,6 +1539,48 @@ fn sibling_merge_scans_have_unique_remote_dyn_filter_producer_ids() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_merge_scan_optimizer_eliminates_projected_false_filter() {
|
||||
init_default_ut_logging();
|
||||
let left_table =
|
||||
TestTable::table_with_filter_pushdown(0, "i1".to_string(), FilterPushDownType::Inexact);
|
||||
let right_table =
|
||||
TestTable::table_with_filter_pushdown(1, "i2".to_string(), FilterPushDownType::Inexact);
|
||||
|
||||
let left_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(left_table),
|
||||
)));
|
||||
let right_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(right_table),
|
||||
)));
|
||||
|
||||
let left = LogicalPlanBuilder::scan_with_filters("i1", left_source, None, vec![])
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
let right = LogicalPlanBuilder::scan_with_filters("i2", right_source, None, vec![])
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let plan = LogicalPlanBuilder::from(left)
|
||||
.cross_join(right)
|
||||
.unwrap()
|
||||
.project(vec![lit(false).alias("cond")])
|
||||
.unwrap()
|
||||
.filter(col("cond"))
|
||||
.unwrap()
|
||||
.sort(vec![col("cond").sort(true, true)])
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let config = ConfigOptions::default();
|
||||
let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
|
||||
|
||||
assert_eq!("EmptyRelation: rows=0", result.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_simplify_now_expression() {
|
||||
init_default_ut_logging();
|
||||
@@ -1543,6 +1664,9 @@ fn expand_proj_limit_part_col_aggr_sort() {
|
||||
|
||||
let config = ConfigOptions::default();
|
||||
let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
|
||||
// Pre-MergeScan optimizer intentionally excludes PushDownLimit, so the
|
||||
// remote plan shows an explicit Limit node instead of `fetch=10` on
|
||||
// TableScan.
|
||||
let expected = [
|
||||
"Sort: t.pk2 ASC NULLS LAST",
|
||||
" Aggregate: groupBy=[[t.pk1, t.pk2]], aggr=[[min(t.number)]]",
|
||||
@@ -1585,6 +1709,9 @@ fn expand_proj_limit_sort_part_col_aggr() {
|
||||
|
||||
let config = ConfigOptions::default();
|
||||
let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
|
||||
// Pre-MergeScan optimizer intentionally excludes PushDownLimit, so the
|
||||
// remote plan shows an explicit Limit node instead of `fetch=10` on
|
||||
// TableScan.
|
||||
let expected = [
|
||||
"Aggregate: groupBy=[[t.pk1, t.pk2]], aggr=[[min(t.number)]]",
|
||||
" Sort: t.pk2 ASC NULLS LAST",
|
||||
@@ -1845,7 +1972,7 @@ fn transform_unalighed_join_with_alias() {
|
||||
let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
|
||||
let expected = [
|
||||
"Limit: skip=0, fetch=1",
|
||||
" LeftSemi Join: Filter: t.number = right.number",
|
||||
" LeftSemi Join: t.number = right.number",
|
||||
" Projection: t.number",
|
||||
" MergeScan [is_placeholder=false, remote_input=[",
|
||||
"TableScan: t",
|
||||
@@ -2321,3 +2448,195 @@ fn scheduled_none_falls_back_to_wall_clock() {
|
||||
"Remote should contain TimestampNanosecond:\n{result_str}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that static side-local predicates on a JOIN input reach the remote
|
||||
/// region TableScan before MergeScan wrapping (issue #8338).
|
||||
///
|
||||
/// Plan shape: Filter(t1.pk1 = 'v') -> Join(t1.number = t2.number) -> TableScan(t1), TableScan(t2)
|
||||
///
|
||||
/// After PushDownFilter runs, the side-local filter should be pushed into the
|
||||
/// left child branch (inside the MergeScan remote_input), making it visible for
|
||||
/// time-index / bloom / skipping pruning.
|
||||
#[test]
|
||||
fn test_join_side_local_filter_pushdown_into_merge_scan() {
|
||||
init_default_ut_logging();
|
||||
let left_table =
|
||||
TestTable::table_with_filter_pushdown(0, "t1".to_string(), FilterPushDownType::Inexact);
|
||||
let right_table =
|
||||
TestTable::table_with_filter_pushdown(1, "t2".to_string(), FilterPushDownType::Inexact);
|
||||
let left_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(left_table),
|
||||
)));
|
||||
let right_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(right_table),
|
||||
)));
|
||||
|
||||
let right_plan = LogicalPlanBuilder::scan_with_filters("t2", right_source, None, vec![])
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Plan: Filter -> Join -> TableScan(left), TableScan(right)
|
||||
let plan = LogicalPlanBuilder::scan_with_filters("t1", left_source, None, vec![])
|
||||
.unwrap()
|
||||
.join_on(
|
||||
right_plan,
|
||||
JoinType::Inner,
|
||||
vec![col("t1.number").eq(col("t2.number"))],
|
||||
)
|
||||
.unwrap()
|
||||
.filter(col("t1.pk1").eq(lit("v"))) // side-local filter on left partition column
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let config = ConfigOptions::default();
|
||||
let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
|
||||
assert_remote_table_scan_filters_are_safe(&result);
|
||||
|
||||
let plan_str = result.to_string();
|
||||
// After PushDownFilter runs, the predicate `t1.pk1 = Utf8("v")` should appear
|
||||
// inside the left MergeScan's remote_input. The pre-MergeScan optimizer may
|
||||
// combine it with join-derived IS NOT NULL pushdowns, so it may not appear as
|
||||
// a standalone Filter: line. It must still be in TableScan partial_filters
|
||||
// and below the Inner Join.
|
||||
assert!(
|
||||
plan_str.contains("t1.pk1 = Utf8(\"v\")"),
|
||||
"Expected predicate t1.pk1 = Utf8(\"v\") in plan, got:\n{plan_str}"
|
||||
);
|
||||
assert!(
|
||||
plan_str.contains(
|
||||
"TableScan: t1, partial_filters=[t1.pk1 = Utf8(\"v\"), t1.number IS NOT NULL]"
|
||||
),
|
||||
"Expected t1 TableScan partial_filters to contain pushed predicate, got:\n{plan_str}"
|
||||
);
|
||||
|
||||
// Find the position of the filter and verify it appears after a MergeScan
|
||||
// opening (i.e., inside remote_input) rather than before the Join.
|
||||
let filter_pos = plan_str
|
||||
.find("TableScan: t1, partial_filters=[t1.pk1 = Utf8(\"v\"), t1.number IS NOT NULL]")
|
||||
.unwrap();
|
||||
let join_pos = plan_str.find("Inner Join").unwrap();
|
||||
// The filter should be after the Join (meaning it was pushed down below the Join,
|
||||
// into a MergeScan's remote_input)
|
||||
assert!(
|
||||
filter_pos > join_pos,
|
||||
"Filter should be pushed below Join (into MergeScan remote_input), but found before Join"
|
||||
);
|
||||
}
|
||||
|
||||
/// LEFT JOIN preserves the left side, so a left-local WHERE predicate is safe
|
||||
/// to push into the left scan before MergeScan wrapping.
|
||||
#[test]
|
||||
fn test_left_join_left_side_filter_pushdown_into_merge_scan() {
|
||||
init_default_ut_logging();
|
||||
let left_table =
|
||||
TestTable::table_with_filter_pushdown(0, "t1".to_string(), FilterPushDownType::Inexact);
|
||||
let right_table =
|
||||
TestTable::table_with_filter_pushdown(1, "t2".to_string(), FilterPushDownType::Inexact);
|
||||
let left_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(left_table),
|
||||
)));
|
||||
let right_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(right_table),
|
||||
)));
|
||||
|
||||
let right_plan = LogicalPlanBuilder::scan_with_filters("t2", right_source, None, vec![])
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let plan = LogicalPlanBuilder::scan_with_filters("t1", left_source, None, vec![])
|
||||
.unwrap()
|
||||
.join_on(
|
||||
right_plan,
|
||||
JoinType::Left,
|
||||
vec![col("t1.number").eq(col("t2.number"))],
|
||||
)
|
||||
.unwrap()
|
||||
.filter(col("t1.pk1").eq(lit("v")))
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let config = ConfigOptions::default();
|
||||
let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
|
||||
assert_remote_table_scan_filters_are_safe(&result);
|
||||
|
||||
let plan_str = result.to_string();
|
||||
assert!(
|
||||
plan_str.contains("TableScan: t1, partial_filters=[t1.pk1 = Utf8(\"v\")]"),
|
||||
"Expected left-side TableScan partial_filters under LEFT JOIN, got:\n{plan_str}"
|
||||
);
|
||||
let scan_filter_pos = plan_str
|
||||
.find("TableScan: t1, partial_filters=[t1.pk1 = Utf8(\"v\")]")
|
||||
.unwrap();
|
||||
let join_pos = plan_str.find("Left Join").unwrap();
|
||||
assert!(
|
||||
scan_filter_pos > join_pos,
|
||||
"Left-side filter should be pushed below LEFT JOIN into MergeScan remote_input:\n{plan_str}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Negative case: cross-table predicate t1.pk1 = t2.pk2 should NOT become a
|
||||
/// side-local scan filter but remain as a join filter.
|
||||
#[test]
|
||||
fn test_join_cross_table_predicate_not_pushed_to_single_side() {
|
||||
init_default_ut_logging();
|
||||
let left_table =
|
||||
TestTable::table_with_filter_pushdown(0, "t1".to_string(), FilterPushDownType::Inexact);
|
||||
let right_table =
|
||||
TestTable::table_with_filter_pushdown(1, "t2".to_string(), FilterPushDownType::Inexact);
|
||||
let left_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(left_table),
|
||||
)));
|
||||
let right_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(right_table),
|
||||
)));
|
||||
|
||||
let right_plan = LogicalPlanBuilder::scan_with_filters("t2", right_source, None, vec![])
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Plan: Filter(t1.pk1 = t2.pk2) -> Join(t1.number = t2.number) -> ...
|
||||
// The filter involves columns from both tables, so PushDownFilter should
|
||||
// keep it as a join filter (not push into a single side's scan).
|
||||
let plan = LogicalPlanBuilder::scan_with_filters("t1", left_source, None, vec![])
|
||||
.unwrap()
|
||||
.join_on(
|
||||
right_plan,
|
||||
JoinType::Inner,
|
||||
vec![col("t1.number").eq(col("t2.number"))],
|
||||
)
|
||||
.unwrap()
|
||||
.filter(col("t1.pk1").eq(col("t2.pk2"))) // cross-table predicate
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let config = ConfigOptions::default();
|
||||
let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
|
||||
assert_remote_table_scan_filters_are_safe(&result);
|
||||
|
||||
let plan_str = result.to_string();
|
||||
// The cross-table predicate should NOT appear as a filter on a single table's
|
||||
// scan inside a MergeScan remote_input. It should remain as part of the
|
||||
// Join's filter.
|
||||
// The key assertion: it should NOT appear as "Filter: t1.pk1 = t2.pk2"
|
||||
assert!(
|
||||
!plan_str.contains("Filter: t1.pk1 = t2.pk2"),
|
||||
"Cross-table predicate should not become a side-local Filter:\n{plan_str}"
|
||||
);
|
||||
assert!(
|
||||
plan_str.contains("t1.pk1 = t2.pk2") || plan_str.contains("t2.pk2 = t1.pk1"),
|
||||
"Cross-table predicate should remain in the join plan:\n{plan_str}"
|
||||
);
|
||||
assert!(
|
||||
!plan_str.contains("partial_filters=[t1.pk1 = t2.pk2]")
|
||||
&& !plan_str.contains("partial_filters=[t2.pk2 = t1.pk1]")
|
||||
&& !plan_str.contains("full_filters=[t1.pk1 = t2.pk2]")
|
||||
&& !plan_str.contains("full_filters=[t2.pk2 = t1.pk1]"),
|
||||
"Cross-table predicate should not become a single-side TableScan filter:\n{plan_str}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -88,12 +88,31 @@ impl PredicateExtractor {
|
||||
Ok(partition_exprs)
|
||||
}
|
||||
|
||||
/// Collect all filter expressions from a logical plan
|
||||
/// Collect all filter expressions from a logical plan.
|
||||
///
|
||||
/// Besides explicit [`LogicalPlan::Filter`] nodes, this must also collect
|
||||
/// predicates already stored in [`LogicalPlan::TableScan`] filters. The
|
||||
/// distributed planner runs a focused DataFusion `PushDownFilter` pass
|
||||
/// before `MergeScan` wrapping, so partition predicates may no longer exist
|
||||
/// as standalone `Filter` nodes by the time region pruning calls this
|
||||
/// extractor. If we ignored `TableScan.filters`, region pruning would miss
|
||||
/// predicates that were successfully pushed down for scan-level pruning.
|
||||
fn collect_filter_expressions(plan: &LogicalPlan, expressions: &mut Vec<Expr>) -> DfResult<()> {
|
||||
if let LogicalPlan::Filter(filter) = plan {
|
||||
expressions.push(filter.predicate.clone());
|
||||
}
|
||||
|
||||
// Collect filters that DataFusion's PushDownFilter stored in TableScan.
|
||||
// `TableScan.filters` is conjunctive: DataFusion passes scan filters as
|
||||
// a list but the table scan must satisfy all of them. Preserve that AND
|
||||
// semantics for partition pruning instead of returning the filters as
|
||||
// independent top-level expressions.
|
||||
if let LogicalPlan::TableScan(table_scan) = plan
|
||||
&& let Some(expr) = Self::conjunction(table_scan.filters.iter().cloned())
|
||||
{
|
||||
expressions.push(expr);
|
||||
}
|
||||
|
||||
// Recursively visit children
|
||||
for child in plan.inputs() {
|
||||
Self::collect_filter_expressions(child, expressions)?;
|
||||
@@ -106,6 +125,11 @@ impl PredicateExtractor {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn conjunction(mut expressions: impl Iterator<Item = Expr>) -> Option<Expr> {
|
||||
let first = expressions.next()?;
|
||||
Some(expressions.fold(first, |acc, expr| acc.and(expr)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of analyzing an expression for partition pruning safety
|
||||
@@ -579,6 +603,69 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extracts_table_scan_filters() {
|
||||
let table_scan = create_test_table_scan();
|
||||
let filter = col("user_id").gt_eq(lit(100i64));
|
||||
let LogicalPlan::TableScan(scan) = table_scan else {
|
||||
panic!("expected test table scan");
|
||||
};
|
||||
let plan = LogicalPlan::TableScan(datafusion_expr::logical_plan::TableScan {
|
||||
filters: vec![filter],
|
||||
..scan
|
||||
});
|
||||
|
||||
let partition_exprs =
|
||||
PredicateExtractor::extract_partition_expressions(&plan, &["user_id".to_string()])
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
partition_exprs,
|
||||
vec![PartitionExpr::new(
|
||||
Operand::Column("user_id".to_string()),
|
||||
RestrictedOp::GtEq,
|
||||
Operand::Value(Value::Int64(100)),
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_combines_table_scan_filters_as_conjunction() {
|
||||
let table_scan = create_test_table_scan();
|
||||
let filter_a = col("user_id").eq(lit(10i64));
|
||||
let filter_b = col("value").eq(lit(20i64));
|
||||
let LogicalPlan::TableScan(scan) = table_scan else {
|
||||
panic!("expected test table scan");
|
||||
};
|
||||
let plan = LogicalPlan::TableScan(datafusion_expr::logical_plan::TableScan {
|
||||
filters: vec![filter_a, filter_b],
|
||||
..scan
|
||||
});
|
||||
|
||||
let partition_exprs = PredicateExtractor::extract_partition_expressions(
|
||||
&plan,
|
||||
&["user_id".to_string(), "value".to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
partition_exprs,
|
||||
vec![PartitionExpr::new(
|
||||
Operand::Expr(PartitionExpr::new(
|
||||
Operand::Column("user_id".to_string()),
|
||||
RestrictedOp::Eq,
|
||||
Operand::Value(Value::Int64(10)),
|
||||
)),
|
||||
RestrictedOp::And,
|
||||
Operand::Expr(PartitionExpr::new(
|
||||
Operand::Column("value".to_string()),
|
||||
RestrictedOp::Eq,
|
||||
Operand::Value(Value::Int64(20)),
|
||||
)),
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_basic_constraints_extraction() {
|
||||
let cases = vec![
|
||||
|
||||
@@ -2465,8 +2465,10 @@ impl PromPlanner {
|
||||
plan: &LogicalPlan,
|
||||
out: &mut BTreeSet<String>,
|
||||
) -> Result<()> {
|
||||
if let LogicalPlan::TableScan(scan) = plan {
|
||||
let table = planner.table_from_source(&scan.source)?;
|
||||
// Derived PromQL plans may contain non-Greptime scans without row-key metadata.
|
||||
if let LogicalPlan::TableScan(scan) = plan
|
||||
&& let Ok(table) = planner.table_from_source(&scan.source)
|
||||
{
|
||||
for col in table.table_info().meta.row_key_column_names() {
|
||||
if col != DATA_SCHEMA_TABLE_ID_COLUMN_NAME
|
||||
&& col != DATA_SCHEMA_TSID_COLUMN_NAME
|
||||
@@ -6576,6 +6578,51 @@ mod test {
|
||||
assert!(!aggr_line.contains(DATA_SCHEMA_TSID_COLUMN_NAME));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aggregate_over_binary_time_function_expr() {
|
||||
for op in ["sum", "min", "max", "avg"] {
|
||||
let prom_expr = parser::parse(&format!(
|
||||
"{op} by (tag_0, tag_1, tag_2) (time() - some_metric)"
|
||||
))
|
||||
.unwrap();
|
||||
let eval_stmt = EvalStmt {
|
||||
expr: prom_expr,
|
||||
start: UNIX_EPOCH,
|
||||
end: UNIX_EPOCH
|
||||
.checked_add(Duration::from_secs(100_000))
|
||||
.unwrap(),
|
||||
interval: Duration::from_secs(5),
|
||||
lookback_delta: Duration::from_secs(1),
|
||||
};
|
||||
|
||||
let table_provider = build_test_table_provider_with_tsid(
|
||||
&[(DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string())],
|
||||
3,
|
||||
1,
|
||||
)
|
||||
.await;
|
||||
let plan =
|
||||
PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let plan_str = plan.display_indent_schema().to_string();
|
||||
let aggr_line = plan_str
|
||||
.lines()
|
||||
.find(|line| line.contains("Aggregate: groupBy="))
|
||||
.unwrap();
|
||||
assert!(aggr_line.contains(op), "{plan_str}");
|
||||
assert!(aggr_line.contains("first_value"), "{plan_str}");
|
||||
assert!(
|
||||
!plan
|
||||
.schema()
|
||||
.fields()
|
||||
.iter()
|
||||
.any(|field| { field.name() == DATA_SCHEMA_TSID_COLUMN_NAME })
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn topk_by_does_not_partition_by_tsid() {
|
||||
let prom_expr = parser::parse("topk by (__tsid) (1, some_metric)").unwrap();
|
||||
|
||||
@@ -1 +1 @@
|
||||
v0.12.2
|
||||
v0.13.6
|
||||
|
||||
@@ -166,8 +166,7 @@ impl GreptimeRequestHandler {
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) =
|
||||
result_sender.try_send(result.map_err(|e| Status::from_error(Box::new(e))))
|
||||
if let Err(e) = result_sender.try_send(result.map_err(Status::from))
|
||||
&& let TrySendError::Closed(_) = e
|
||||
{
|
||||
warn!(r#""DoPut" client maybe unreachable, abort handling its message"#);
|
||||
@@ -298,13 +297,18 @@ impl Drop for RequestTimer {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::FixedOffset;
|
||||
use common_error::GREPTIME_DB_HEADER_ERROR_CODE;
|
||||
use common_error::ext::BoxedError;
|
||||
use common_time::Timezone;
|
||||
use query::options::FLOW_SCHEDULED_TIME_MILLIS;
|
||||
use session::hints::{
|
||||
INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY, REMOTE_QUERY_ID_EXTENSION_KEY,
|
||||
};
|
||||
use snafu::ResultExt;
|
||||
use tonic::Code;
|
||||
|
||||
use super::*;
|
||||
use crate::error::{ExecuteGrpcRequestSnafu, InvalidParameterSnafu};
|
||||
|
||||
#[test]
|
||||
fn test_create_query_context() {
|
||||
@@ -378,4 +382,34 @@ mod tests {
|
||||
query_context.remote_query_id()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_batch_error_to_status_preserves_error_details() {
|
||||
let inner = InvalidParameterSnafu {
|
||||
reason: "Column not found, column: new_col",
|
||||
}
|
||||
.build();
|
||||
let err = Err::<(), _>(BoxedError::new(inner))
|
||||
.context(ExecuteGrpcRequestSnafu)
|
||||
.unwrap_err();
|
||||
|
||||
let status = Status::from(err);
|
||||
|
||||
assert_eq!(status.code(), Code::InvalidArgument);
|
||||
assert!(
|
||||
status
|
||||
.message()
|
||||
.contains("Column not found, column: new_col")
|
||||
);
|
||||
assert!(
|
||||
status
|
||||
.message()
|
||||
.contains("Invalid request parameter: Column not found")
|
||||
);
|
||||
assert!(
|
||||
status
|
||||
.metadata()
|
||||
.contains_key(GREPTIME_DB_HEADER_ERROR_CODE)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +119,7 @@ pub mod test_helpers;
|
||||
|
||||
pub const HTTP_API_VERSION: &str = "v1";
|
||||
pub const HTTP_API_PREFIX: &str = "/v1/";
|
||||
pub const HTTP_API_PREFIX_WITHOUT_TRAILING_SLASH: &str = "/v1";
|
||||
/// Default http body limit (64M).
|
||||
const DEFAULT_BODY_LIMIT: ReadableSize = ReadableSize::mb(64);
|
||||
|
||||
|
||||
@@ -25,16 +25,19 @@ use common_telemetry::warn;
|
||||
///
|
||||
/// Extracts client address from [`ConnectInfo`] if available.
|
||||
pub async fn log_error_with_client_ip(req: Request<Body>, next: Next) -> Response {
|
||||
let request_info = req
|
||||
.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|c| c.0)
|
||||
.map(|addr| {
|
||||
let method = req.method().clone();
|
||||
let uri = req.uri().clone();
|
||||
let matched_path = req.extensions().get::<MatchedPath>().cloned();
|
||||
(addr, method, uri, matched_path)
|
||||
});
|
||||
let request_info = if is_public_http_api_path(req.uri().path()) {
|
||||
req.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|c| c.0)
|
||||
.map(|addr| {
|
||||
let method = req.method().clone();
|
||||
let uri = req.uri().clone();
|
||||
let matched_path = req.extensions().get::<MatchedPath>().cloned();
|
||||
(addr, method, uri, matched_path)
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let response = next.run(req).await;
|
||||
|
||||
@@ -57,6 +60,11 @@ pub async fn log_error_with_client_ip(req: Request<Body>, next: Next) -> Respons
|
||||
response
|
||||
}
|
||||
|
||||
fn is_public_http_api_path(path: &str) -> bool {
|
||||
path == super::HTTP_API_PREFIX_WITHOUT_TRAILING_SLASH
|
||||
|| path.starts_with(super::HTTP_API_PREFIX)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::Router;
|
||||
@@ -66,6 +74,19 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_public_http_api_path_matches_v1_prefix() {
|
||||
assert!(is_public_http_api_path("/v1"));
|
||||
assert!(is_public_http_api_path("/v1/sql"));
|
||||
assert!(is_public_http_api_path("/v1/prometheus/api/v1/query"));
|
||||
|
||||
assert!(!is_public_http_api_path("/"));
|
||||
assert!(!is_public_http_api_path("/health"));
|
||||
assert!(!is_public_http_api_path("/status"));
|
||||
assert!(!is_public_http_api_path("/metrics"));
|
||||
assert!(!is_public_http_api_path("/v10/sql"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_middleware_passes_error_response() {
|
||||
async fn not_found_handler() -> StatusCode {
|
||||
|
||||
@@ -43,8 +43,8 @@ use datafusion_common::ScalarValue;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, SchemaRef};
|
||||
use datatypes::types::jsonb_to_string;
|
||||
use futures::StreamExt;
|
||||
use futures::future::join_all;
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use itertools::Itertools;
|
||||
use promql_parser::label::{METRIC_NAME, MatchOp, Matcher, Matchers};
|
||||
use promql_parser::parser::token::{self};
|
||||
@@ -624,13 +624,9 @@ async fn get_all_column_names(
|
||||
schema: &str,
|
||||
manager: &CatalogManagerRef,
|
||||
) -> std::result::Result<HashSet<String>, catalog::error::Error> {
|
||||
let table_names = manager.table_names(catalog, schema, None).await?;
|
||||
|
||||
let mut labels = HashSet::new();
|
||||
for table_name in table_names {
|
||||
let Some(table) = manager.table(catalog, schema, &table_name, None).await? else {
|
||||
continue;
|
||||
};
|
||||
let mut tables = manager.tables(catalog, schema, None);
|
||||
while let Some(table) = tables.try_next().await? {
|
||||
for column in table.primary_key_columns() {
|
||||
if column.name != DATA_SCHEMA_TABLE_ID_COLUMN_NAME
|
||||
&& column.name != DATA_SCHEMA_TSID_COLUMN_NAME
|
||||
@@ -1683,7 +1679,16 @@ pub async fn parse_query(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use catalog::memory::MemoryCatalogManager;
|
||||
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use promql_parser::parser::value::ValueType;
|
||||
use table::metadata::{TableInfoBuilder, TableMetaBuilder, TableType, TableVersion};
|
||||
use table::test_util::EmptyTable;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -1895,4 +1900,52 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_all_column_names_uses_tag_columns() {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
ColumnSchema::new(
|
||||
"greptime_timestamp",
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
)
|
||||
.with_time_index(true),
|
||||
ColumnSchema::new("host", ConcreteDataType::string_datatype(), false),
|
||||
ColumnSchema::new("region", ConcreteDataType::string_datatype(), false),
|
||||
ColumnSchema::new("value", ConcreteDataType::float64_datatype(), true),
|
||||
ColumnSchema::new(
|
||||
DATA_SCHEMA_TSID_COLUMN_NAME,
|
||||
ConcreteDataType::uint64_datatype(),
|
||||
true,
|
||||
),
|
||||
]));
|
||||
let meta = TableMetaBuilder::empty()
|
||||
.schema(schema)
|
||||
.primary_key_indices(vec![1, 2, 4])
|
||||
.engine("metric".to_string())
|
||||
.next_column_id(5)
|
||||
.build()
|
||||
.unwrap();
|
||||
let table_info = TableInfoBuilder::default()
|
||||
.table_id(1024)
|
||||
.table_version(0 as TableVersion)
|
||||
.name("cpu_usage")
|
||||
.catalog_name(DEFAULT_CATALOG_NAME)
|
||||
.schema_name(DEFAULT_SCHEMA_NAME)
|
||||
.table_type(TableType::Base)
|
||||
.meta(meta)
|
||||
.build()
|
||||
.unwrap();
|
||||
let manager: CatalogManagerRef =
|
||||
MemoryCatalogManager::new_with_table(EmptyTable::from_table_info(&table_info));
|
||||
|
||||
let labels = get_all_column_names(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, &manager)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
labels,
|
||||
HashSet::from(["host".to_string(), "region".to_string()])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,7 +108,8 @@ pub struct FileRefsManifest {
|
||||
|
||||
#[derive(Clone, Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct GcReport {
|
||||
/// deleted files per region
|
||||
/// Deleted SST/parquet file ids per region. Index-only deletions are reported via
|
||||
/// `deleted_indexes` because a naked `FileId` cannot distinguish index versions.
|
||||
/// TODO(discord9): change to `RemovedFile`?
|
||||
pub deleted_files: HashMap<RegionId, Vec<FileId>>,
|
||||
pub deleted_indexes: HashMap<RegionId, Vec<(FileId, IndexVersion)>>,
|
||||
|
||||
@@ -22,6 +22,7 @@ use datafusion::catalog::Session;
|
||||
use datafusion::datasource::{TableProvider, TableType as DfTableType};
|
||||
use datafusion::error::Result as DfResult;
|
||||
use datafusion::physical_plan::ExecutionPlan;
|
||||
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
|
||||
use datafusion_expr::TableProviderFilterPushDown as DfTableProviderFilterPushDown;
|
||||
use datafusion_expr::expr::Expr;
|
||||
use datafusion_physical_expr::PhysicalSortExpr;
|
||||
@@ -139,10 +140,76 @@ impl TableProvider for DfTableProviderAdapter {
|
||||
&self,
|
||||
filters: &[&Expr],
|
||||
) -> DfResult<Vec<DfTableProviderFilterPushDown>> {
|
||||
let schema = self.schema();
|
||||
let filters = filters.iter().map(|&x| x.clone()).collect::<Vec<_>>();
|
||||
Ok(self
|
||||
.table
|
||||
.supports_filters_pushdown(&filters.iter().collect::<Vec<_>>())
|
||||
.map(|v| v.into_iter().map(Into::into).collect::<Vec<_>>())?)
|
||||
.map(|v| {
|
||||
v.into_iter()
|
||||
.zip(filters.iter())
|
||||
.map(|(ty, expr)| {
|
||||
if !is_scan_local(expr, &schema) {
|
||||
DfTableProviderFilterPushDown::Unsupported
|
||||
} else {
|
||||
ty.into()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the expression can be safely evaluated by a remote scan.
|
||||
/// Rejects outer references and column references unknown to the schema.
|
||||
fn is_scan_local(expr: &Expr, schema: &DfSchemaRef) -> bool {
|
||||
let mut problems = false;
|
||||
let _ = expr.apply(|node| match node {
|
||||
Expr::OuterReferenceColumn(_, _) => {
|
||||
problems = true;
|
||||
Ok(TreeNodeRecursion::Stop)
|
||||
}
|
||||
Expr::Column(col) => {
|
||||
if schema.column_with_name(&col.name).is_none() {
|
||||
problems = true;
|
||||
return Ok(TreeNodeRecursion::Stop);
|
||||
}
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
}
|
||||
_ => Ok(TreeNodeRecursion::Continue),
|
||||
});
|
||||
!problems
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use datafusion_common::Column as DfColumn;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_is_scan_local_normal_column() {
|
||||
use datafusion::arrow::datatypes::{DataType, Field, Schema};
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));
|
||||
let expr = Expr::Column(DfColumn::new(Some("t"), "x"));
|
||||
assert!(is_scan_local(&expr, &schema));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_scan_local_unknown_column() {
|
||||
use datafusion::arrow::datatypes::{DataType, Field, Schema};
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));
|
||||
let expr = Expr::Column(DfColumn::new(Some("t"), "z"));
|
||||
assert!(!is_scan_local(&expr, &schema));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_scan_local_outer_ref() {
|
||||
use datafusion::arrow::datatypes::Schema;
|
||||
use datatypes::arrow::datatypes::{DataType, Field};
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));
|
||||
let field = Arc::new(Field::new("x", DataType::Int64, true));
|
||||
let expr = Expr::OuterReferenceColumn(field, DfColumn::new(Some("t"), "x"));
|
||||
assert!(!is_scan_local(&expr, &schema));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,11 +17,11 @@ use std::time::Duration;
|
||||
|
||||
use auth::user_provider_from_option;
|
||||
use chrono::{DateTime, NaiveDate, NaiveDateTime, SecondsFormat, Utc};
|
||||
use common_catalog::consts::DEFAULT_PRIVATE_SCHEMA_NAME;
|
||||
use common_catalog::consts::{DEFAULT_PRIVATE_SCHEMA_NAME, DEFAULT_SCHEMA_NAME};
|
||||
use common_frontend::slow_query_event::{
|
||||
SLOW_QUERY_TABLE_COST_COLUMN_NAME, SLOW_QUERY_TABLE_IS_PROMQL_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_NAME, SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME, SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME,
|
||||
};
|
||||
use sqlx::mysql::{MySqlConnection, MySqlDatabaseError, MySqlPoolOptions};
|
||||
use sqlx::postgres::{PgDatabaseError, PgPoolOptions};
|
||||
@@ -720,10 +720,11 @@ pub async fn test_mysql_slow_query(store_type: StorageType) {
|
||||
|
||||
let table = format!("{}.{}", DEFAULT_PRIVATE_SCHEMA_NAME, SLOW_QUERY_TABLE_NAME);
|
||||
let query = format!(
|
||||
"SELECT {}, {}, {}, {} FROM {table} WHERE {} = ?",
|
||||
"SELECT {}, {}, {}, {}, {} FROM {table} WHERE {} = ?",
|
||||
SLOW_QUERY_TABLE_COST_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_IS_PROMQL_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
|
||||
);
|
||||
@@ -747,10 +748,12 @@ pub async fn test_mysql_slow_query(store_type: StorageType) {
|
||||
let cost: u64 = row.get(0);
|
||||
let threshold: u64 = row.get(1);
|
||||
let query: String = row.get(2);
|
||||
let is_promql: bool = row.get(3);
|
||||
let schema_name: String = row.get(3);
|
||||
let is_promql: bool = row.get(4);
|
||||
|
||||
assert!(cost > 0 && threshold > 0 && cost > threshold);
|
||||
assert_eq!(query, slow_query);
|
||||
assert_eq!(schema_name, DEFAULT_SCHEMA_NAME);
|
||||
assert!(!is_promql);
|
||||
|
||||
let _ = fe_mysql_server.shutdown().await;
|
||||
@@ -847,10 +850,11 @@ pub async fn test_postgres_slow_query(store_type: StorageType) {
|
||||
|
||||
let table = format!("{}.{}", DEFAULT_PRIVATE_SCHEMA_NAME, SLOW_QUERY_TABLE_NAME);
|
||||
let query = format!(
|
||||
"SELECT {}, {}, {}, {} FROM {table} WHERE {} = $1",
|
||||
"SELECT {}, {}, {}, {}, {} FROM {table} WHERE {} = $1",
|
||||
SLOW_QUERY_TABLE_COST_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_IS_PROMQL_COLUMN_NAME,
|
||||
SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
|
||||
);
|
||||
@@ -873,10 +877,12 @@ pub async fn test_postgres_slow_query(store_type: StorageType) {
|
||||
let cost: Decimal = row.get(0);
|
||||
let threshold: Decimal = row.get(1);
|
||||
let query: String = row.get(2);
|
||||
let is_promql: bool = row.get(3);
|
||||
let schema_name: String = row.get(3);
|
||||
let is_promql: bool = row.get(4);
|
||||
|
||||
assert!(cost > 0.into() && threshold > 0.into() && cost > threshold);
|
||||
assert_eq!(query, slow_query);
|
||||
assert_eq!(schema_name, DEFAULT_SCHEMA_NAME);
|
||||
assert!(!is_promql);
|
||||
|
||||
let _ = fe_pg_server.shutdown().await;
|
||||
|
||||
@@ -83,37 +83,45 @@ limit 1;
|
||||
|_|_Inner Join: t_3.ts = t_4.ts, t_3.vin = t_4.vin_|
|
||||
|_|_Inner Join: t_2.ts = t_3.ts, t_2.vin = t_3.vin_|
|
||||
|_|_Inner Join: t_1.ts = t_2.ts, t_1.vin = t_2.vin_|
|
||||
|_|_Filter: t_1.vin IS NOT NULL_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_1_|
|
||||
|_| ]]_|
|
||||
|_|_Filter: t_2.vin IS NOT NULL_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_2_|
|
||||
|_| Filter: t_1.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_1, partial_filters=[t_1.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_3_|
|
||||
|_| Filter: t_2.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_2, partial_filters=[t_2.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_4_|
|
||||
|_| Filter: t_3.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_3, partial_filters=[t_3.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_5_|
|
||||
|_| Filter: t_4.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_4, partial_filters=[t_4.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_6_|
|
||||
|_| Filter: t_5.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_5, partial_filters=[t_5.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_7_|
|
||||
|_| Filter: t_6.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_6, partial_filters=[t_6.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_8_|
|
||||
|_| Filter: t_7.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_7, partial_filters=[t_7.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_9_|
|
||||
|_| Filter: t_8.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_8, partial_filters=[t_8.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: t_10_|
|
||||
|_| Filter: t_9.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_9, partial_filters=[t_9.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| Filter: t_10.vin IS NOT NULL_|
|
||||
|_|_TableScan: t_10, partial_filters=[t_10.vin IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | SortPreservingMergeExec: [ts@0 DESC], fetch=1_|
|
||||
|_|_SortExec: TopK(fetch=1), expr=[ts@0 DESC], preserve_partitioning=[true]_|
|
||||
@@ -127,10 +135,8 @@ limit 1;
|
||||
|_|_REDACTED
|
||||
|_|_REDACTED
|
||||
|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_FilterExec: vin@1 IS NOT NULL_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_FilterExec: vin@1 IS NOT NULL_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|
||||
@@ -28,7 +28,7 @@ explain SELECT * FROM multi_partitions_test_table WHERE ts > cast(1000000000 as
|
||||
|_| Sort: multi_partitions_test_table.host ASC NULLS LAST_|
|
||||
|_|_Projection: multi_partitions_test_table.host, multi_partitions_test_table.ts, multi_partitions_test_table.cpu, multi_partitions_test_table.memory, multi_partitions_test_table.disk_util |
|
||||
|_|_Filter: multi_partitions_test_table.ts > TimestampMillisecond(1000000000, None)_|
|
||||
|_|_TableScan: multi_partitions_test_table_|
|
||||
|_|_TableScan: multi_partitions_test_table, partial_filters=[multi_partitions_test_table.ts > TimestampMillisecond(1000000000, None)]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | SortPreservingMergeExec: [host@0 ASC NULLS LAST]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|
||||
@@ -35,7 +35,7 @@ tql explain (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr
|
||||
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
|
||||
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
|
||||
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
|
||||
| | TableScan: aggr_optimize_not |
|
||||
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] |
|
||||
| | MergeScanExec: REDACTED
|
||||
@@ -104,7 +104,7 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_op
|
||||
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
|
||||
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
|
||||
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
|
||||
| | TableScan: aggr_optimize_not |
|
||||
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST] |
|
||||
| | SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
@@ -173,7 +173,7 @@ tql explain (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optim
|
||||
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
|
||||
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
|
||||
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
|
||||
| | TableScan: aggr_optimize_not |
|
||||
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST] |
|
||||
| | SortExec: expr=[a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
@@ -242,7 +242,7 @@ tql explain (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time
|
||||
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
|
||||
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
|
||||
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
|
||||
| | TableScan: aggr_optimize_not |
|
||||
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST] |
|
||||
| | MergeScanExec: REDACTED
|
||||
@@ -307,7 +307,7 @@ tql explain (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr
|
||||
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
|
||||
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
|
||||
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
|
||||
| | TableScan: aggr_optimize_not |
|
||||
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] |
|
||||
| | SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
@@ -373,7 +373,7 @@ tql explain sum(aggr_optimize_not);
|
||||
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
|
||||
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
|
||||
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(-299999, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(0, None) |
|
||||
| | TableScan: aggr_optimize_not |
|
||||
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(-299999, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(0, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [greptime_timestamp@0 ASC NULLS LAST] |
|
||||
| | SortExec: expr=[greptime_timestamp@0 ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
@@ -459,7 +459,7 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize
|
||||
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
|
||||
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
|
||||
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
|
||||
| | TableScan: aggr_optimize_not |
|
||||
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
|
||||
| | ]] |
|
||||
| | SubqueryAlias: aggr_optimize_not_count |
|
||||
| | Sort: aggr_optimize_not_count.a ASC NULLS LAST, aggr_optimize_not_count.b ASC NULLS LAST, aggr_optimize_not_count.c ASC NULLS LAST, aggr_optimize_not_count.greptime_timestamp ASC NULLS LAST |
|
||||
@@ -472,7 +472,7 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize
|
||||
| | Sort: aggr_optimize_not_count.a ASC NULLS FIRST, aggr_optimize_not_count.b ASC NULLS FIRST, aggr_optimize_not_count.c ASC NULLS FIRST, aggr_optimize_not_count.d ASC NULLS FIRST, aggr_optimize_not_count.greptime_timestamp ASC NULLS FIRST |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Filter: aggr_optimize_not_count.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not_count.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
|
||||
| | TableScan: aggr_optimize_not_count |
|
||||
| | TableScan: aggr_optimize_not_count, partial_filters=[aggr_optimize_not_count.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not_count.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@5 / sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@4 as aggr_optimize_not.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000))) / aggr_optimize_not_count.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] |
|
||||
| | REDACTED
|
||||
@@ -1159,7 +1159,7 @@ EXPLAIN SELECT SUM(val_col_1), COUNT(*) FROM step_aggr_extended WHERE pk_col_1 =
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| Aggregate: groupBy=[[]], aggr=[[__sum_state(step_aggr_extended.val_col_1), __count_state(step_aggr_extended.ts)]]_|
|
||||
|_|_Filter: step_aggr_extended.pk_col_1 = Utf8("non_existent")_|
|
||||
|_|_TableScan: step_aggr_extended_|
|
||||
|_|_TableScan: step_aggr_extended, partial_filters=[step_aggr_extended.pk_col_1 = Utf8("non_existent")]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | ProjectionExec: expr=[sum(step_aggr_extended.val_col_1)@0 as sum(step_aggr_extended.val_col_1), count(Int64(1))@1 as count(*)]_|
|
||||
|_|_AggregateExec: mode=Final, gby=[], aggr=[sum(step_aggr_extended.val_col_1), count(Int64(1))]_|
|
||||
|
||||
@@ -250,7 +250,7 @@ GROUP BY
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| Aggregate: groupBy=[[base_table.env, base_table.service_name, base_table.city, base_table.page, CAST(date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"), base_table.time) AS Timestamp(s)) AS arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))]], aggr=[[__uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)]]_|
|
||||
|_|_Filter: (base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) OR base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) OR base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) OR base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) OR base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) OR base_table.fid > Int64(0) AND base_table.fid < Int64(3000000)) AND base_table.time >= TimestampMillisecond(0, None)_|
|
||||
|_|_TableScan: base_table_|
|
||||
|_|_TableScan: base_table, partial_filters=[base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) OR base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) OR base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) OR base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) OR base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) OR base_table.fid > Int64(0) AND base_table.fid < Int64(3000000), base_table.time >= TimestampMillisecond(0, None)]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | ProjectionExec: expr=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@5 as lcp_state, max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@6 as max_lcp, min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@7 as min_lcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@8 as fmp_state, max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@9 as max_fmp, min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@10 as min_fmp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@11 as fcp_state, max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@12 as max_fcp, min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@13 as min_fcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@14 as fp_state, max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@15 as max_fp, min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@16 as min_fp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@17 as tti_state, max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@18 as max_tti, min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@19 as min_tti, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@20 as fid_state, max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@21 as max_fid, min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@22 as min_fid, max(base_table.shard_key)@23 as shard_key, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))]_|
|
||||
|_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(base_table.shard_key)]_|
|
||||
@@ -597,7 +597,7 @@ where
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| Aggregate: groupBy=[[]], aggr=[[__count_state(base_table.time)]]_|
|
||||
|_|_Filter: base_table.time >= TimestampMillisecond(0, None)_|
|
||||
|_|_TableScan: base_table_|
|
||||
|_|_TableScan: base_table, partial_filters=[base_table.time >= TimestampMillisecond(0, None)]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | ProjectionExec: expr=[count(Int64(1))@0 as count(*)]_|
|
||||
|_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]_|
|
||||
|
||||
@@ -15,12 +15,14 @@ EXPLAIN SELECT * FROM integers WHERE i IN ((SELECT i FROM integers)) ORDER BY i;
|
||||
| logical_plan_| Sort: integers.i ASC NULLS LAST_|
|
||||
|_|_LeftSemi Join: integers.i = __correlated_sq_1.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: integers_|
|
||||
|_| Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| Projection: integers.i_|
|
||||
|_|_TableScan: integers_|
|
||||
|_| SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_Projection: integers.i_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | SortPreservingMergeExec: [i@0 ASC NULLS LAST]_|
|
||||
|_|_SortExec: expr=[i@0 ASC NULLS LAST], preserve_partitioning=[true]_|
|
||||
@@ -46,12 +48,14 @@ EXPLAIN SELECT * FROM integers i1 WHERE EXISTS(SELECT i FROM integers WHERE i=i1
|
||||
|_|_LeftSemi Join: i1.i = __correlated_sq_1.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| SubqueryAlias: i1_|
|
||||
|_|_TableScan: integers_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_Projection: integers.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: integers_|
|
||||
|_| SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_Projection: integers.i_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | SortPreservingMergeExec: [i@0 ASC NULLS LAST]_|
|
||||
|_|_SortExec: expr=[i@0 ASC NULLS LAST], preserve_partitioning=[true]_|
|
||||
@@ -59,7 +63,6 @@ EXPLAIN SELECT * FROM integers i1 WHERE EXISTS(SELECT i FROM integers WHERE i=i1
|
||||
|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
+-+-+
|
||||
@@ -86,10 +89,10 @@ order by t.i desc;
|
||||
| logical_plan_| Sort: t.i DESC NULLS FIRST_|
|
||||
|_|_SubqueryAlias: t_|
|
||||
|_|_Cross Join:_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_Projection: integers.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: integers_|
|
||||
|_| Filter: integers.i IS NOT NULL AND Boolean(true)_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL, Boolean(true)] |
|
||||
|_| ]]_|
|
||||
|_|_Projection:_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
@@ -99,7 +102,6 @@ order by t.i desc;
|
||||
|_|_SortExec: expr=[i@0 DESC], preserve_partitioning=[true]_|
|
||||
|_|_CrossJoinExec_|
|
||||
|_|_CoalescePartitionsExec_|
|
||||
|_|_FilterExec: i@0 IS NOT NULL_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[]_|
|
||||
@@ -125,11 +127,11 @@ EXPLAIN INSERT INTO other SELECT i, 2 FROM integers WHERE i=(SELECT MAX(i) FROM
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | TableScan: integers |
|
||||
| | ]] |
|
||||
| | SubqueryAlias: __scalar_sq_1 |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: max(integers.i) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[max(integers.i)]] |
|
||||
| | TableScan: integers |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: __scalar_sq_1 |
|
||||
| | Projection: max(integers.i) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[max(integers.i)]] |
|
||||
| | TableScan: integers, partial_filters=[Boolean(true)] |
|
||||
| | ]] |
|
||||
| physical_plan_error | This feature is not implemented: Insert into not implemented for this table |
|
||||
+---------------------+-----------------------------------------------------------------------------+
|
||||
@@ -165,18 +167,19 @@ EXPLAIN SELECT * FROM integers i1 WHERE EXISTS(SELECT i FROM integers WHERE i=i1
|
||||
|_|_LeftSemi Join: i1.i = __correlated_sq_1.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| SubqueryAlias: i1_|
|
||||
|_|_TableScan: integers_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_Projection: integers.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: integers_|
|
||||
|_| SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_Projection: integers.i_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | SortPreservingMergeExec: [i@0 ASC NULLS LAST]_|
|
||||
|_|_SortExec: expr=[i@0 ASC NULLS LAST], preserve_partitioning=[true]_|
|
||||
|_|_REDACTED
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
+-+-+
|
||||
@@ -195,20 +198,22 @@ EXPLAIN SELECT * FROM integers i1 WHERE EXISTS(SELECT count(i) FROM integers WHE
|
||||
|_|_LeftSemi Join: i1.i = __correlated_sq_1.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| SubqueryAlias: i1_|
|
||||
|_|_TableScan: integers_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_Aggregate: groupBy=[[integers.i]], aggr=[[]]_|
|
||||
|_|_Projection: integers.i_|
|
||||
|_|_Projection: __correlated_sq_1.i_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: integers_|
|
||||
|_| SubqueryAlias: __correlated_sq_1_|
|
||||
|_|_Projection: count(integers.i), integers.i_|
|
||||
|_|_Aggregate: groupBy=[[integers.i]], aggr=[[count(integers.i)]]_|
|
||||
|_|_Filter: integers.i IS NOT NULL_|
|
||||
|_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | SortPreservingMergeExec: [i@0 ASC NULLS LAST]_|
|
||||
|_|_SortExec: expr=[i@0 ASC NULLS LAST], preserve_partitioning=[true]_|
|
||||
|_|_REDACTED
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_AggregateExec: mode=SinglePartitioned, gby=[i@0 as i], aggr=[]_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
+-+-+
|
||||
@@ -543,12 +548,14 @@ ORDER BY u1.a;
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| SubqueryAlias: u1_|
|
||||
|_|_Projection: t1.a_|
|
||||
|_|_TableScan: t1_|
|
||||
|_|_Filter: t1.a IS NOT NULL_|
|
||||
|_|_TableScan: t1, partial_filters=[t1.a IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| SubqueryAlias: u2_|
|
||||
|_|_Projection: t2.a_|
|
||||
|_|_TableScan: t2_|
|
||||
|_|_Filter: t2.a IS NOT NULL_|
|
||||
|_|_TableScan: t2, partial_filters=[t2.a IS NOT NULL]_|
|
||||
|_| ]]_|
|
||||
| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST]_|
|
||||
|_|_SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]_|
|
||||
|
||||
@@ -100,31 +100,30 @@ TQL EXPLAIN (
|
||||
)
|
||||
);
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | HistogramFold: le=le, field=sum(prom_avg_over_time(ts_range,v)), quantile=0.5 |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Sort: test_tsid.le ASC NULLS LAST, test_tsid.tag4 ASC NULLS LAST, test_tsid.tag5 ASC NULLS LAST, test_tsid.ts ASC NULLS LAST |
|
||||
| | Aggregate: groupBy=[[test_tsid.le, test_tsid.tag4, test_tsid.tag5, test_tsid.ts]], aggr=[[sum(prom_avg_over_time(ts_range,v))]] |
|
||||
| | Filter: prom_avg_over_time(ts_range,v) IS NOT NULL |
|
||||
| | Projection: test_tsid.ts, prom_avg_over_time(ts_range, v) AS prom_avg_over_time(ts_range,v), test_tsid.le, test_tsid.tag1, test_tsid.tag2, test_tsid.tag4, test_tsid.tag5, test_tsid.tag6, test_tsid.tag7, test_tsid.tag8, test_tsid.__tsid |
|
||||
| | PromRangeManipulate: req range=[1769139000000..1769139900000], interval=[60000], eval range=[1800000], time index=[ts], values=["v"] |
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=["__tsid"] |
|
||||
| | Sort: test_tsid.__tsid ASC NULLS FIRST, test_tsid.ts ASC NULLS FIRST |
|
||||
| | Filter: test_tsid.ts >= TimestampMillisecond(1769137200001, None) AND test_tsid.ts <= TimestampMillisecond(1769139900000, None) |
|
||||
| | Projection: test_tsid.v, test_tsid.le, test_tsid.tag1, test_tsid.tag2, test_tsid.tag4, test_tsid.tag5, test_tsid.tag6, test_tsid.tag7, test_tsid.tag8, test_tsid.__tsid, test_tsid.ts |
|
||||
| | SubqueryAlias: test_tsid |
|
||||
| | Filter: phy.__table_id=UInt32(REDACTED) |
|
||||
| | TableScan: phy projection=[ts, v, tag1, tag2, le, tag4, tag5, tag6, tag7, tag8, __table_id, __tsid] |
|
||||
| | ]] |
|
||||
| physical_plan | HistogramFoldExec: le=@0, field=@4, quantile=0.5 |
|
||||
| | SortExec: expr=[tag4@1 ASC NULLS LAST, tag5@2 ASC NULLS LAST, ts@3 ASC NULLS LAST, CAST(le@0 AS Float64) ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | HistogramFold: le=le, field=sum(prom_avg_over_time(ts_range,v)), quantile=0.5 |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Sort: test_tsid.le ASC NULLS LAST, test_tsid.tag4 ASC NULLS LAST, test_tsid.tag5 ASC NULLS LAST, test_tsid.ts ASC NULLS LAST |
|
||||
| | Aggregate: groupBy=[[test_tsid.le, test_tsid.tag4, test_tsid.tag5, test_tsid.ts]], aggr=[[sum(prom_avg_over_time(ts_range,v))]] |
|
||||
| | Filter: prom_avg_over_time(ts_range,v) IS NOT NULL |
|
||||
| | Projection: test_tsid.ts, prom_avg_over_time(ts_range, v) AS prom_avg_over_time(ts_range,v), test_tsid.le, test_tsid.tag1, test_tsid.tag2, test_tsid.tag4, test_tsid.tag5, test_tsid.tag6, test_tsid.tag7, test_tsid.tag8, test_tsid.__tsid |
|
||||
| | PromRangeManipulate: req range=[1769139000000..1769139900000], interval=[60000], eval range=[1800000], time index=[ts], values=["v"] |
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=["__tsid"] |
|
||||
| | Sort: test_tsid.__tsid ASC NULLS FIRST, test_tsid.ts ASC NULLS FIRST |
|
||||
| | Projection: test_tsid.v, test_tsid.le, test_tsid.tag1, test_tsid.tag2, test_tsid.tag4, test_tsid.tag5, test_tsid.tag6, test_tsid.tag7, test_tsid.tag8, test_tsid.__tsid, test_tsid.ts |
|
||||
| | SubqueryAlias: test_tsid |
|
||||
| | Filter: phy.ts >= TimestampMillisecond(1769137200001, None) AND phy.ts <= TimestampMillisecond(1769139900000, None) AND phy.__table_id=UInt32(REDACTED) |
|
||||
| | TableScan: phy projection=[ts, v, tag1, tag2, le, tag4, tag5, tag6, tag7, tag8, __table_id, __tsid], partial_filters=[phy.ts >= TimestampMillisecond(1769137200001, None), phy.ts <= TimestampMillisecond(1769139900000, None), phy.__table_id=UInt32(REDACTED)] |
|
||||
| | ]] |
|
||||
| physical_plan | HistogramFoldExec: le=@0, field=@4, quantile=0.5 |
|
||||
| | SortExec: expr=[tag4@1 ASC NULLS LAST, tag5@2 ASC NULLS LAST, ts@3 ASC NULLS LAST, CAST(le@0 AS Float64) ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
| | RepartitionExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
CREATE FLOW IF NOT EXISTS test_tsid
|
||||
SINK TO 'test_tsid_output'
|
||||
|
||||
@@ -42,22 +42,22 @@ WHERE c.tier = 'gold';
|
||||
| | Projection: o.id, o.customer_id, o.amount |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: o |
|
||||
| | TableScan: orders |
|
||||
| | Filter: orders.customer_id IS NOT NULL |
|
||||
| | TableScan: orders, partial_filters=[orders.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| | Filter: c.tier = Utf8("gold") |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | TableScan: customers |
|
||||
| | Filter: customers.customer_id IS NOT NULL AND customers.tier = Utf8("gold") |
|
||||
| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold"), customers.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] |
|
||||
| | RepartitionExec: REDACTED
|
||||
| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: REDACTED
|
||||
| | FilterExec: tier@2 = gold |
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
@@ -91,15 +91,16 @@ WHERE c.tier = 'gold';
|
||||
|_|_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_FilterExec: tier@2 = gold metrics=REDACTED_|
|
||||
|_|_|_ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@1 IS NOT NULL metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND tier@2 = gold metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
|_|_| Total rows: REDACTED_|
|
||||
+-+-+-+
|
||||
@@ -160,33 +161,33 @@ FROM (SELECT "id", customer_id as cid, amount, ts FROM orders) o
|
||||
JOIN customers c ON o.cid = c.customer_id
|
||||
WHERE c.tier IN ('gold', 'silver');
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Projection: o.id, o.amount, c.name, c.tier |
|
||||
| | Inner Join: o.cid = c.customer_id |
|
||||
| | Projection: o.id, o.cid, o.amount |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: o |
|
||||
| | Projection: orders.id, orders.customer_id AS cid, orders.amount, orders.ts |
|
||||
| | TableScan: orders |
|
||||
| | ]] |
|
||||
| | Filter: c.tier = Utf8("gold") OR c.tier = Utf8("silver") |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | TableScan: customers |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cid@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Projection: o.id, o.amount, c.name, c.tier |
|
||||
| | Inner Join: o.cid = c.customer_id |
|
||||
| | Projection: o.id, o.cid, o.amount |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: o |
|
||||
| | Projection: orders.id, orders.customer_id AS cid, orders.amount, orders.ts |
|
||||
| | Filter: orders.customer_id IS NOT NULL |
|
||||
| | TableScan: orders, partial_filters=[orders.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")) |
|
||||
| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("silver"), customers.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cid@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] |
|
||||
| | RepartitionExec: REDACTED
|
||||
| | ProjectionExec: expr=[id@0 as id, cid@1 as cid, amount@2 as amount] |
|
||||
| | ProjectionExec: expr=[id@0 as id, cid@1 as cid, amount@2 as amount] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: REDACTED
|
||||
| | FilterExec: tier@2 = gold OR tier@2 = silver |
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------+
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- SQLNESS REPLACE ("metrics_per_partition":\s*.*metrics=) "metrics_per_partition": REDACTED metrics=
|
||||
-- SQLNESS REPLACE (metrics=\{.*\}) metrics=REDACTED
|
||||
@@ -218,16 +219,17 @@ WHERE c.tier IN ('gold', 'silver');
|
||||
|_|_|_ProjectionExec: expr=[id@0 as id, cid@1 as cid, amount@2 as amount] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_FilterExec: tier@2 = gold OR tier@2 = silver metrics=REDACTED_|
|
||||
|_|_|_ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as cid, amount@2 as amount, ts@3 as ts] metrics=REDACTED_|
|
||||
|_|_|_FilterExec: customer_id@1 IS NOT NULL metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
|_|_| Total rows: REDACTED_|
|
||||
+-+-+-+
|
||||
|
||||
@@ -47,35 +47,35 @@ FROM (
|
||||
JOIN customers c ON top_orders.customer_id = c.customer_id
|
||||
WHERE c.tier IN ('gold', 'bronze');
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Projection: top_orders.id, top_orders.amount, c.name, c.tier |
|
||||
| | Inner Join: top_orders.customer_id = c.customer_id |
|
||||
| | Projection: top_orders.id, top_orders.customer_id, top_orders.amount |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: top_orders |
|
||||
| | Limit: skip=0, fetch=5 |
|
||||
| | Sort: orders.amount DESC NULLS FIRST |
|
||||
| | Projection: orders.id, orders.customer_id, orders.amount, orders.ts |
|
||||
| | TableScan: orders |
|
||||
| | ]] |
|
||||
| | Filter: c.tier = Utf8("gold") OR c.tier = Utf8("bronze") |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | TableScan: customers |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Projection: top_orders.id, top_orders.amount, c.name, c.tier |
|
||||
| | Inner Join: top_orders.customer_id = c.customer_id |
|
||||
| | Projection: top_orders.id, top_orders.customer_id, top_orders.amount |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: top_orders |
|
||||
| | Filter: orders.customer_id IS NOT NULL |
|
||||
| | Limit: skip=0, fetch=5 |
|
||||
| | Sort: orders.amount DESC NULLS FIRST |
|
||||
| | Projection: orders.id, orders.customer_id, orders.amount, orders.ts |
|
||||
| | TableScan: orders |
|
||||
| | ]] |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("bronze")) |
|
||||
| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("bronze"), customers.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] |
|
||||
| | RepartitionExec: partitioning=REDACTED
|
||||
| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] |
|
||||
| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: partitioning=REDACTED
|
||||
| | FilterExec: tier@2 = gold OR tier@2 = bronze |
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- SQLNESS REPLACE ("metrics_per_partition":\s*.*metrics=) "metrics_per_partition": REDACTED metrics=
|
||||
-- SQLNESS REPLACE (metrics=\{.*\}) metrics=REDACTED
|
||||
@@ -112,16 +112,18 @@ WHERE c.tier IN ('gold', 'bronze');
|
||||
|_|_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_FilterExec: tier@2 = gold OR tier@2 = bronze metrics=REDACTED_|
|
||||
|_|_|_ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_SortPreservingMergeExec: [amount@2 DESC], fetch=5 metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@1 IS NOT NULL metrics=REDACTED_|
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_SortPreservingMergeExec: [amount@2 DESC], fetch=5 metrics=REDACTED_|
|
||||
|_|_|_SortExec: TopK(fetch=5), expr=[amount@2 DESC], preserve_partitioning=[true] metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = bronze) metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"bronze\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
|_|_| Total rows: REDACTED_|
|
||||
+-+-+-+
|
||||
@@ -211,13 +213,14 @@ LIMIT 4;
|
||||
| | Projection: o.id, o.customer_id, o.amount |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: o |
|
||||
| | TableScan: orders |
|
||||
| | Filter: orders.customer_id IS NOT NULL |
|
||||
| | TableScan: orders, partial_filters=[orders.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| | Filter: c.tier = Utf8("gold") OR c.tier = Utf8("silver") |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | TableScan: customers |
|
||||
| | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")) |
|
||||
| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("silver"), customers.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [amount@4 DESC], fetch=4 |
|
||||
| | SortExec: TopK(fetch=4), expr=[amount@4 DESC], preserve_partitioning=[true] |
|
||||
@@ -227,9 +230,8 @@ LIMIT 4;
|
||||
| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: partitioning=REDACTED
|
||||
| | FilterExec: tier@2 = gold OR tier@2 = silver |
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
@@ -271,15 +273,16 @@ LIMIT 4;
|
||||
|_|_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_FilterExec: tier@2 = gold OR tier@2 = silver metrics=REDACTED_|
|
||||
|_|_|_ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@1 IS NOT NULL metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
|_|_| Total rows: REDACTED_|
|
||||
+-+-+-+
|
||||
@@ -377,19 +380,20 @@ WHERE c.tier IN ('gold', 'silver')
|
||||
| | Projection: o.id, o.customer_id, o.product_id, o.amount |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: o |
|
||||
| | TableScan: orders |
|
||||
| | Filter: orders.product_id IS NOT NULL AND orders.customer_id IS NOT NULL |
|
||||
| | TableScan: orders, partial_filters=[orders.product_id IS NOT NULL, orders.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| | Filter: c.tier = Utf8("gold") OR c.tier = Utf8("silver") |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: c.customer_id, c.name, c.tier |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | TableScan: customers |
|
||||
| | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")) |
|
||||
| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("silver"), customers.customer_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| | Filter: p.category = Utf8("electronics") |
|
||||
| | Projection: p.product_id, p.name, p.category |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: p.product_id, p.name, p.category |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: p |
|
||||
| | TableScan: products |
|
||||
| | Filter: products.product_id IS NOT NULL AND products.category = Utf8("electronics") |
|
||||
| | TableScan: products, partial_filters=[products.category = Utf8("electronics"), products.product_id IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | ProjectionExec: expr=[id@0 as id, amount@1 as amount, name@2 as name, tier@3 as tier, name@4 as product_name, category@5 as category] |
|
||||
| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(product_id@1, product_id@0)], projection=[id@0, amount@2, name@3, tier@4, name@6, category@7] |
|
||||
@@ -399,13 +403,11 @@ WHERE c.tier IN ('gold', 'silver')
|
||||
| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, product_id@2 as product_id, amount@3 as amount] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: partitioning=REDACTED
|
||||
| | FilterExec: tier@2 = gold OR tier@2 = silver |
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: partitioning=REDACTED
|
||||
| | FilterExec: category@2 = electronics |
|
||||
| | ProjectionExec: expr=[product_id@0 as product_id, name@1 as name, category@2 as category] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | ProjectionExec: expr=[product_id@0 as product_id, name@1 as name, category@2 as category] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
@@ -444,22 +446,23 @@ WHERE c.tier IN ('gold', 'silver')
|
||||
|_|_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, product_id@2 as product_id, amount@3 as amount] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_FilterExec: tier@2 = gold OR tier@2 = silver metrics=REDACTED_|
|
||||
|_|_|_ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_FilterExec: category@2 = electronics metrics=REDACTED_|
|
||||
|_|_|_ProjectionExec: expr=[product_id@0 as product_id, name@1 as name, category@2 as category] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: product_id@2 IS NOT NULL AND customer_id@1 IS NOT NULL metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "filters": ["product_id IS NOT NULL", "customer_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: product_id@0 IS NOT NULL AND category@2 = electronics metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "filters": ["category = Utf8(\"electronics\")", "product_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_|
|
||||
|_|_|_|
|
||||
|_|_| Total rows: REDACTED_|
|
||||
+-+-+-+
|
||||
|
||||
@@ -147,10 +147,12 @@ LIMIT 1;
|
||||
|_|_|_ProjectionExec: expr=[vec_id@0 as vec_id] metrics=REDACTED_|
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: vec_id@0 IS NOT NULL metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec metrics=REDACTED_|
|
||||
| 1_| 0_|_FilterExec: vec_id@0 IS NOT NULL metrics=REDACTED_|
|
||||
|_|_|_CooperativeExec metrics=REDACTED_|
|
||||
|_|_|_SeqScan: REDACTED
|
||||
|_|_|_|
|
||||
|_|_| Total rows: REDACTED_|
|
||||
|
||||
@@ -16,26 +16,26 @@ tql explain (0, 100, '1s')
|
||||
tag_a="ffa",
|
||||
}[1h])[12h:1h];
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromRangeManipulate: req range=[0..100000], interval=[1000], eval range=[43200000], time index=[ts], values=["prom_increase(ts_range,val,ts,Int64(3600000))"] |
|
||||
| | PromSeriesDivide: tags=["tag_a", "tag_b"] |
|
||||
| | Sort: count_total.tag_a ASC NULLS FIRST, count_total.tag_b ASC NULLS FIRST, count_total.ts ASC NULLS FIRST |
|
||||
| | Filter: prom_increase(ts_range,val,ts,Int64(3600000)) IS NOT NULL |
|
||||
| | Projection: count_total.ts, prom_increase(ts_range, val, count_total.ts, Int64(3600000)) AS prom_increase(ts_range,val,ts,Int64(3600000)), count_total.tag_a, count_total.tag_b |
|
||||
| | PromRangeManipulate: req range=[-39600000..100000], interval=[3600000], eval range=[3600000], time index=[ts], values=["val"] |
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=["tag_a", "tag_b"] |
|
||||
| | Sort: count_total.tag_a ASC NULLS FIRST, count_total.tag_b ASC NULLS FIRST, count_total.ts ASC NULLS FIRST |
|
||||
| | Filter: count_total.tag_a = Utf8("ffa") AND count_total.ts >= TimestampMillisecond(-43199999, None) AND count_total.ts <= TimestampMillisecond(100000, None) |
|
||||
| | TableScan: count_total |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromRangeManipulate: req range=[0..100000], interval=[1000], eval range=[43200000], time index=[ts], values=["prom_increase(ts_range,val,ts,Int64(3600000))"] |
|
||||
| | PromSeriesDivide: tags=["tag_a", "tag_b"] |
|
||||
| | Sort: count_total.tag_a ASC NULLS FIRST, count_total.tag_b ASC NULLS FIRST, count_total.ts ASC NULLS FIRST |
|
||||
| | Filter: prom_increase(ts_range,val,ts,Int64(3600000)) IS NOT NULL |
|
||||
| | Projection: count_total.ts, prom_increase(ts_range, val, count_total.ts, Int64(3600000)) AS prom_increase(ts_range,val,ts,Int64(3600000)), count_total.tag_a, count_total.tag_b |
|
||||
| | PromRangeManipulate: req range=[-39600000..100000], interval=[3600000], eval range=[3600000], time index=[ts], values=["val"] |
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=["tag_a", "tag_b"] |
|
||||
| | Sort: count_total.tag_a ASC NULLS FIRST, count_total.tag_b ASC NULLS FIRST, count_total.ts ASC NULLS FIRST |
|
||||
| | Filter: count_total.tag_a = Utf8("ffa") AND count_total.ts >= TimestampMillisecond(-43199999, None) AND count_total.ts <= TimestampMillisecond(100000, None) |
|
||||
| | TableScan: count_total, partial_filters=[count_total.tag_a = Utf8("ffa"), count_total.ts >= TimestampMillisecond(-43199999, None), count_total.ts <= TimestampMillisecond(100000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
tql eval (0, 100, '1s')
|
||||
increase(count_total{
|
||||
|
||||
@@ -117,10 +117,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".*"};
|
||||
+-+-+-+
|
||||
| stage | node | plan_|
|
||||
+-+-+-+
|
||||
| 0_| 0_|_CooperativeExec REDACTED
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED
|
||||
| 0_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_PromSeriesDivideExec: tags=["host"] REDACTED
|
||||
|_|_|_SortExec: expr=[host@1 ASC, ts@0 ASC], preserve_partitioning=[false] REDACTED
|
||||
|
||||
@@ -691,6 +691,19 @@ TQL EVAL (0, 5, '5s') rate(tsid_binary_join_left[5s]) / tsid_binary_join_left;
|
||||
+------+-----+----+----------------------------------------------------------------------------+
|
||||
+------+-----+----+----------------------------------------------------------------------------+
|
||||
|
||||
-- Regression for aggregating a binary expression over `time()` and a TSID-backed metric.
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
TQL EVAL (0, 5, '5s') sum by (host, job) (time() - tsid_binary_join_left);
|
||||
|
||||
+-------+------+---------------------+-------------------------------------------------------------------+
|
||||
| host | job | ts | sum(.time / Float64(1000) - tsid_binary_join_left.greptime_value) |
|
||||
+-------+------+---------------------+-------------------------------------------------------------------+
|
||||
| host1 | job1 | 1970-01-01T00:00:00 | -12.0 |
|
||||
| host1 | job1 | 1970-01-01T00:00:05 | -10.0 |
|
||||
| host2 | job2 | 1970-01-01T00:00:00 | -18.0 |
|
||||
| host2 | job2 | 1970-01-01T00:00:05 | -16.0 |
|
||||
+-------+------+---------------------+-------------------------------------------------------------------+
|
||||
|
||||
DROP TABLE tsid_binary_join_third;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -252,6 +252,10 @@ TQL EVAL (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsid_b
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
TQL EVAL (0, 5, '5s') rate(tsid_binary_join_left[5s]) / tsid_binary_join_left;
|
||||
|
||||
-- Regression for aggregating a binary expression over `time()` and a TSID-backed metric.
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
TQL EVAL (0, 5, '5s') sum by (host, job) (time() - tsid_binary_join_left);
|
||||
|
||||
DROP TABLE tsid_binary_join_third;
|
||||
DROP TABLE tsid_binary_join_right_by_job;
|
||||
DROP TABLE tsid_binary_join_right;
|
||||
|
||||
@@ -55,7 +55,7 @@ WHERE s."id"=e.sid
|
||||
AND e.grade <= (SELECT AVG(e2.grade) - 1 FROM exams e2 WHERE s."id"=e2.sid OR (e2.curriculum=s.major AND s."year">=e2."year"))
|
||||
ORDER BY "name", course;
|
||||
|
||||
Error: 1001(Unsupported), This feature is not implemented: Physical plan does not support logical expression ScalarSubquery(<subquery>)
|
||||
Error: 3001(EngineExecuteQuery), Error during planning: Correlated scalar subquery can only be used in Projection, Filter, Aggregate plan nodes
|
||||
|
||||
-- Test 3: EXISTS subquery
|
||||
SELECT "name", major
|
||||
|
||||
@@ -12,20 +12,20 @@ Affected Rows: 3
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
TQL EXPLAIN (0, 10, '5s') test;
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST |
|
||||
| | Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None) |
|
||||
| | TableScan: test |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST |
|
||||
| | Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None) |
|
||||
| | TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- 'lookback' parameter is not fully supported, the test has to be updated
|
||||
-- explain at 0s, 5s and 10s. No point at 0s.
|
||||
@@ -33,40 +33,40 @@ TQL EXPLAIN (0, 10, '5s') test;
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
TQL EXPLAIN (0, 10, '1s', '2s') test;
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromInstantManipulate: range=[0..10000], lookback=[2000], interval=[1000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST |
|
||||
| | Filter: test.j >= TimestampMillisecond(-1999, None) AND test.j <= TimestampMillisecond(10000, None) |
|
||||
| | TableScan: test |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromInstantManipulate: range=[0..10000], lookback=[2000], interval=[1000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST |
|
||||
| | Filter: test.j >= TimestampMillisecond(-1999, None) AND test.j <= TimestampMillisecond(10000, None) |
|
||||
| | TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-1999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- explain at 0s, 5s and 10s. No point at 0s.
|
||||
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
TQL EXPLAIN ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test;
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST |
|
||||
| | Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None) |
|
||||
| | TableScan: test |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST |
|
||||
| | Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None) |
|
||||
| | TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- explain verbose at 0s, 5s and 10s. No point at 0s.
|
||||
-- SQLNESS REPLACE (-+) -
|
||||
@@ -98,7 +98,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test;
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST_|
|
||||
|_|_Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_|
|
||||
|_|_TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| logical_plan after FixStateUdafOrderingAnalyzer_| SAME TEXT AS ABOVE_|
|
||||
| analyzed_logical_plan_| SAME TEXT AS ABOVE_|
|
||||
@@ -130,7 +130,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test;
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST_|
|
||||
|_|_Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_|
|
||||
|_|_TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_|
|
||||
@@ -165,7 +165,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test;
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST_|
|
||||
|_|_Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_|
|
||||
|_|_TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| initial_physical_plan_| MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
@@ -247,7 +247,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series;
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST_|
|
||||
|_|_Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_|
|
||||
|_|_TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| logical_plan after FixStateUdafOrderingAnalyzer_| SAME TEXT AS ABOVE_|
|
||||
| analyzed_logical_plan_| SAME TEXT AS ABOVE_|
|
||||
@@ -280,7 +280,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series;
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST_|
|
||||
|_|_Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_|
|
||||
|_|_TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_|
|
||||
@@ -316,7 +316,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series;
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test.k ASC NULLS FIRST, test.j ASC NULLS FIRST_|
|
||||
|_|_Filter: test.j >= TimestampMillisecond(-299999, None) AND test.j <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_|
|
||||
|_|_TableScan: test, partial_filters=[test.j >= TimestampMillisecond(-299999, None), test.j <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| initial_physical_plan_| MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
@@ -380,29 +380,25 @@ Affected Rows: 3
|
||||
-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED
|
||||
TQL EXPLAIN (0, 10, '5s') test_nano;
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST |
|
||||
| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j |
|
||||
| | Projection: test_nano.i, test_nano.j, test_nano.k |
|
||||
| | Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None) |
|
||||
| | Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | TableScan: test_nano |
|
||||
| | ]] |
|
||||
| physical_plan | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivideExec: tags=["k"] |
|
||||
| | SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true] |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivide: tags=["k"] |
|
||||
| | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST |
|
||||
| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None) |
|
||||
| | TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] |
|
||||
| | PromSeriesDivideExec: tags=["k"] |
|
||||
| | SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true] |
|
||||
| | RepartitionExec: REDACTED
|
||||
| | ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j] |
|
||||
| | FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3] |
|
||||
| | ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- explain verbose at 0s, 5s and 10s for a nanosecond time index.
|
||||
-- SQLNESS REPLACE (-+) -
|
||||
@@ -432,11 +428,11 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
| logical_plan after DistPlannerAnalyzer_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Filter: test_nano.j >= TimestampMillisecond(-299999, None) AND test_nano.j <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| logical_plan after FixStateUdafOrderingAnalyzer_| SAME TEXT AS ABOVE_|
|
||||
| analyzed_logical_plan_| SAME TEXT AS ABOVE_|
|
||||
@@ -457,39 +453,19 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_filter_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after push_down_filter_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_group_by_constant_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after common_sub_expression_eliminate_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: __common_expr_1 >= TimestampMillisecond(-299999, None) AND __common_expr_1 <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: __common_expr_1 >= TimestampMillisecond(-299999, None) AND __common_expr_1 <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_|
|
||||
@@ -510,134 +486,39 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_filter_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after push_down_filter_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_group_by_constant_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after common_sub_expression_eliminate_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: __common_expr_2 >= TimestampMillisecond(-299999, None) AND __common_expr_2 <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: __common_expr_2 >= TimestampMillisecond(-299999, None) AND __common_expr_2 <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after optimize_unions_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_join_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after decorrelate_predicate_subquery_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after scalar_subquery_to_join_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after decorrelate_lateral_join_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after extract_equijoin_predicate_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_duplicated_expr_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_filter_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_cross_join_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_limit_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after propagate_empty_relation_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_filter_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_|
|
||||
| logical_plan after eliminate_group_by_constant_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after common_sub_expression_eliminate_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| ]]_|
|
||||
| logical_plan after optimize_projections_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_|
|
||||
| logical_plan_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivide: tags=["k"]_|
|
||||
|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_|
|
||||
|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_|
|
||||
|_|_Projection: test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None)_|
|
||||
|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k_|
|
||||
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|
||||
|_| TableScan: test_nano_|
|
||||
|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_|
|
||||
|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] |
|
||||
|_| ]]_|
|
||||
| initial_physical_plan_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivideExec: tags=["k"]_|
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| initial_physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]]_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] |
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| initial_physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_|
|
||||
|_|_PromSeriesDivideExec: tags=["k"], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_|
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k], schema=[i:Float64;N, j:Timestamp(ns), k:Utf8;N]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, schema=[__common_expr_3:Timestamp(ms), i:Float64;N, j:Timestamp(ns), k:Utf8;N]_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], schema=[__common_expr_3:Timestamp(ms), i:Float64;N, j:Timestamp(ns), k:Utf8;N]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan after OutputRequirements_| OutputRequirementExec: order_by=[], dist_by=Unspecified_|
|
||||
@@ -645,9 +526,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
|_|_PromSeriesDivideExec: tags=["k"]_|
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan after aggregate_statistics_| SAME TEXT AS ABOVE_|
|
||||
@@ -662,9 +540,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
|_|_PromSeriesDivideExec: tags=["k"]_|
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan after EnforceDistribution_| OutputRequirementExec: order_by=[], dist_by=Unspecified_|
|
||||
@@ -674,9 +549,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
|_|_RepartitionExec: REDACTED
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_|
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_|
|
||||
@@ -686,29 +558,15 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_|
|
||||
|_|_RepartitionExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan after OptimizeAggregateOrder_| SAME TEXT AS ABOVE_|
|
||||
| physical_plan after ProjectionPushdown_| OutputRequirementExec: order_by=[], dist_by=Unspecified_|
|
||||
|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivideExec: tags=["k"]_|
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_|
|
||||
|_|_RepartitionExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3]_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_|
|
||||
| physical_plan after OutputRequirements_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_|
|
||||
|_|_PromSeriesDivideExec: tags=["k"]_|
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_|
|
||||
|_|_RepartitionExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3]_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_|
|
||||
@@ -727,8 +585,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_|
|
||||
|_|_RepartitionExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3]_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
@@ -736,8 +592,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_RepartitionExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] |
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
| physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_|
|
||||
@@ -745,8 +599,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano;
|
||||
|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_|
|
||||
|_|_RepartitionExec: REDACTED
|
||||
|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_|
|
||||
|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3], schema=[i:Float64;N, j:Timestamp(ns), k:Utf8;N]_|
|
||||
|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], schema=[__common_expr_3:Timestamp(ms), i:Float64;N, j:Timestamp(ns), k:Utf8;N]_|
|
||||
|_|_MergeScanExec: REDACTED
|
||||
|_|_|
|
||||
+-+-+
|
||||
|
||||
@@ -35,10 +35,9 @@ TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9;
|
||||
|_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[5000], time index=[ts] REDACTED
|
||||
|_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED
|
||||
|_|_|_PromSeriesDivideExec: tags=["job"] REDACTED
|
||||
|_|_|_SortExec: expr=[ts@2 ASC], preserve_partitioning=[true] REDACTED
|
||||
|_|_|_SortExec: expr=[job@1 ASC, ts@2 ASC], preserve_partitioning=[true] REDACTED
|
||||
|_|_|_RepartitionExec: partitioning=REDACTED
|
||||
|_|_|_ProjectionExec: expr=[value@1 as value, job@0 as job, CAST(ts@2 AS Timestamp(ms)) as ts] REDACTED
|
||||
|_|_|_FilterExec: job@0 = fire AND ts@2 >= -4999000000 AND ts@2 <= 10000000000 REDACTED
|
||||
|_|_|_MergeScanExec: REDACTED
|
||||
|_|_|_|
|
||||
| 1_| 0_|_CooperativeExec REDACTED
|
||||
|
||||
@@ -52,21 +52,21 @@ EXPLAIN WITH tql as (
|
||||
)
|
||||
SELECT * FROM tql;
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: tql.ts, tql.val |
|
||||
| | SubqueryAlias: tql |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: tql.ts, tql.val |
|
||||
| | SubqueryAlias: tql |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with column aliases
|
||||
WITH tql (the_timestamp, the_value) as (
|
||||
@@ -91,22 +91,22 @@ EXPLAIN WITH tql (the_timestamp, the_value) as (
|
||||
)
|
||||
SELECT * FROM tql;
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: tql.the_timestamp, tql.the_value |
|
||||
| | SubqueryAlias: tql |
|
||||
| | Projection: metric.ts AS the_timestamp, metric.val AS the_value |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: tql.the_timestamp, tql.the_value |
|
||||
| | SubqueryAlias: tql |
|
||||
| | Projection: metric.ts AS the_timestamp, metric.val AS the_value |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- Explain TQL CTE
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
@@ -115,21 +115,21 @@ EXPLAIN WITH tql AS (
|
||||
TQL EVAL (0, 40, '10s') metric
|
||||
) SELECT * FROM tql;
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: tql.ts, tql.val |
|
||||
| | SubqueryAlias: tql |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: tql.ts, tql.val |
|
||||
| | SubqueryAlias: tql |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- Hybrid CTEs (TQL + SQL)
|
||||
WITH
|
||||
@@ -150,26 +150,26 @@ EXPLAIN WITH
|
||||
filtered AS (SELECT * FROM tql_data WHERE val > 5)
|
||||
SELECT count(*) FROM filtered;
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] |
|
||||
| | SubqueryAlias: filtered |
|
||||
| | Projection: tql_data.ts, tql_data.val |
|
||||
| | Filter: tql_data.val > Float64(5) |
|
||||
| | SubqueryAlias: tql_data |
|
||||
| | Projection: metric.ts AS ts, metric.val AS val |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] |
|
||||
| | SubqueryAlias: filtered |
|
||||
| | Projection: tql_data.ts, tql_data.val |
|
||||
| | SubqueryAlias: tql_data |
|
||||
| | Projection: metric.ts AS ts, metric.val AS val |
|
||||
| | Filter: metric.val > Float64(5) |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with complex PromQL expressions
|
||||
WITH
|
||||
@@ -190,30 +190,29 @@ EXPLAIN WITH
|
||||
filtered (ts, val) AS (SELECT * FROM tql_data WHERE val > 0)
|
||||
SELECT sum(val) FROM filtered;
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: sum(filtered.val) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[sum(filtered.val)]] |
|
||||
| | SubqueryAlias: filtered |
|
||||
| | Projection: tql_data.ts AS ts, tql_data.val AS val |
|
||||
| | Projection: tql_data.ts, tql_data.val |
|
||||
| | Filter: tql_data.val > Float64(0) |
|
||||
| | SubqueryAlias: tql_data |
|
||||
| | Projection: metric.ts AS ts, prom_rate(ts_range,val,ts,Int64(20000)) AS val |
|
||||
| | Filter: prom_rate(ts_range,val,ts,Int64(20000)) IS NOT NULL |
|
||||
| | Projection: metric.ts, prom_rate(ts_range, val, metric.ts, Int64(20000)) AS prom_rate(ts_range,val,ts,Int64(20000)) |
|
||||
| | PromRangeManipulate: req range=[0..40000], interval=[10000], eval range=[20000], time index=[ts], values=["val"] |
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-19999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: sum(filtered.val) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[sum(filtered.val)]] |
|
||||
| | SubqueryAlias: filtered |
|
||||
| | Projection: tql_data.ts AS ts, tql_data.val AS val |
|
||||
| | Projection: tql_data.ts, tql_data.val |
|
||||
| | SubqueryAlias: tql_data |
|
||||
| | Projection: metric.ts AS ts, prom_rate(ts_range,val,ts,Int64(20000)) AS val |
|
||||
| | Filter: prom_rate(ts_range,val,ts,Int64(20000)) > Float64(0) AND prom_rate(ts_range,val,ts,Int64(20000)) IS NOT NULL |
|
||||
| | Projection: metric.ts, prom_rate(ts_range, val, metric.ts, Int64(20000)) AS prom_rate(ts_range,val,ts,Int64(20000)) |
|
||||
| | PromRangeManipulate: req range=[0..40000], interval=[10000], eval range=[20000], time index=[ts], values=["val"] |
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-19999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-19999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with aggregation functions
|
||||
WITH tql_agg(ts, summary) AS (
|
||||
@@ -234,24 +233,24 @@ EXPLAIN WITH tql_agg(ts, summary) AS (
|
||||
)
|
||||
SELECT round(avg(summary)) as avg_sum FROM tql_agg;
|
||||
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: round(avg(tql_agg.summary)) AS avg_sum |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[avg(tql_agg.summary)]] |
|
||||
| | SubqueryAlias: tql_agg |
|
||||
| | Projection: labels.ts AS ts, sum(labels.cpu) AS summary |
|
||||
| | Aggregate: groupBy=[[labels.ts]], aggr=[[sum(labels.cpu)]] |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host ~ Utf8("^(?:host.*)$") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: round(avg(tql_agg.summary)) AS avg_sum |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[avg(tql_agg.summary)]] |
|
||||
| | SubqueryAlias: tql_agg |
|
||||
| | Projection: labels.ts AS ts, sum(labels.cpu) AS summary |
|
||||
| | Aggregate: groupBy=[[labels.ts]], aggr=[[sum(labels.cpu)]] |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host ~ Utf8("^(?:host.*)$") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels, partial_filters=[labels.host ~ Utf8("^(?:host.*)$"), labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with label selectors
|
||||
WITH host_metrics AS (
|
||||
@@ -272,22 +271,22 @@ EXPLAIN WITH host_metrics AS (
|
||||
)
|
||||
SELECT count(*) as host1_points FROM host_metrics;
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) AS host1_points |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(host_metrics.ts) AS count(Int64(1))]] |
|
||||
| | SubqueryAlias: host_metrics |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host1") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) AS host1_points |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(host_metrics.ts) AS count(Int64(1))]] |
|
||||
| | SubqueryAlias: host_metrics |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host1") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels, partial_filters=[labels.host = Utf8("host1"), labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with column reference
|
||||
WITH host_metrics AS (
|
||||
@@ -312,21 +311,21 @@ EXPLAIN WITH host_metrics AS (
|
||||
)
|
||||
SELECT host_metrics.ts, host_metrics.host FROM host_metrics;
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: host_metrics.ts, host_metrics.host |
|
||||
| | SubqueryAlias: host_metrics |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host1") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: host_metrics.ts, host_metrics.host |
|
||||
| | SubqueryAlias: host_metrics |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host1") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels, partial_filters=[labels.host = Utf8("host1"), labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- Multiple TQL CTEs referencing different tables
|
||||
WITH
|
||||
@@ -361,44 +360,44 @@ WHERE m.ts = l.ts
|
||||
ORDER BY m.ts
|
||||
LIMIT 3;
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Projection: metric_val, label_val |
|
||||
| | Sort: m.ts ASC NULLS LAST, fetch=3 |
|
||||
| | Projection: m.val AS metric_val, l.cpu AS label_val, m.ts |
|
||||
| | Inner Join: m.ts = l.ts |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: m |
|
||||
| | SubqueryAlias: metric_data |
|
||||
| | Projection: metric.ts AS ts, metric.val AS val |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| | Projection: l.ts, l.cpu |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: l |
|
||||
| | SubqueryAlias: label_data |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host2") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | ]] |
|
||||
| physical_plan | ProjectionExec: expr=[metric_val@0 as metric_val, label_val@1 as label_val] |
|
||||
| | SortPreservingMergeExec: [ts@2 ASC NULLS LAST], fetch=3 |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Projection: metric_val, label_val |
|
||||
| | Sort: m.ts ASC NULLS LAST, fetch=3 |
|
||||
| | Projection: m.val AS metric_val, l.cpu AS label_val, m.ts |
|
||||
| | Inner Join: m.ts = l.ts |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: m |
|
||||
| | SubqueryAlias: metric_data |
|
||||
| | Projection: metric.ts AS ts, metric.val AS val |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| | Projection: l.ts, l.cpu |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: l |
|
||||
| | SubqueryAlias: label_data |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host2") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels, partial_filters=[labels.host = Utf8("host2"), labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | ProjectionExec: expr=[metric_val@0 as metric_val, label_val@1 as label_val] |
|
||||
| | SortPreservingMergeExec: [ts@2 ASC NULLS LAST], fetch=3 |
|
||||
| | SortExec: TopK(fetch=3), expr=[ts@2 ASC NULLS LAST], preserve_REDACTED
|
||||
| | ProjectionExec: expr=[val@1 as metric_val, cpu@2 as label_val, ts@0 as ts] |
|
||||
| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ts@0, ts@0)], projection=[ts@0, val@1, cpu@3] |
|
||||
| | ProjectionExec: expr=[val@1 as metric_val, cpu@2 as label_val, ts@0 as ts] |
|
||||
| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ts@0, ts@0)], projection=[ts@0, val@1, cpu@3] |
|
||||
| | RepartitionExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: REDACTED
|
||||
| | ProjectionExec: expr=[ts@0 as ts, cpu@2 as cpu] |
|
||||
| | ProjectionExec: expr=[ts@0 as ts, cpu@2 as cpu] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with mathematical operations
|
||||
WITH computed(ts, val) AS (
|
||||
@@ -419,25 +418,25 @@ EXPLAIN WITH computed(ts, val) AS (
|
||||
)
|
||||
SELECT min(val) as min_computed, max(val) as max_computed FROM computed;
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: min(computed.val) AS min_computed, max(computed.val) AS max_computed |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[min(computed.val), max(computed.val)]] |
|
||||
| | SubqueryAlias: computed |
|
||||
| | Projection: metric.ts AS ts, val * Float64(2) + Float64(1) AS val |
|
||||
| | Projection: metric.ts, CAST(val * Float64(2) AS Float64) + Float64(1) AS val * Float64(2) + Float64(1) |
|
||||
| | Projection: metric.ts, CAST(metric.val AS Float64) * Float64(2) AS val * Float64(2) |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: min(computed.val) AS min_computed, max(computed.val) AS max_computed |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[min(computed.val), max(computed.val)]] |
|
||||
| | SubqueryAlias: computed |
|
||||
| | Projection: metric.ts AS ts, val * Float64(2) + Float64(1) AS val |
|
||||
| | Projection: metric.ts, CAST(val * Float64(2) AS Float64) + Float64(1) AS val * Float64(2) + Float64(1) |
|
||||
| | Projection: metric.ts, CAST(metric.val AS Float64) * Float64(2) AS val * Float64(2) |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with window functions in SQL part
|
||||
WITH tql_base(ts, val) AS (
|
||||
@@ -484,7 +483,7 @@ ORDER BY ts;
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | ProjectionExec: expr=[ts@0 as ts, val@1 as val, lag(tql_base.val,Int64(1)) ORDER BY [tql_base.ts ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as prev_value] |
|
||||
| | BoundedWindowAggExec: wdw=[lag(tql_base.val,Int64(1)) ORDER BY [tql_base.ts ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "lag(tql_base.val,Int64(1)) ORDER BY [tql_base.ts ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Float64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] |
|
||||
@@ -523,24 +522,24 @@ FROM tql_grouped
|
||||
GROUP BY minute
|
||||
HAVING count(*) > 1;
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: date_trunc(Utf8("minute"),tql_grouped.ts) AS minute, count(Int64(1)) AS count(*) AS point_count |
|
||||
| | Filter: count(Int64(1)) > Int64(1) |
|
||||
| | Aggregate: groupBy=[[date_trunc(Utf8("minute"), tql_grouped.ts)]], aggr=[[count(tql_grouped.ts) AS count(Int64(1))]] |
|
||||
| | SubqueryAlias: tql_grouped |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: date_trunc(Utf8("minute"),tql_grouped.ts) AS minute, count(Int64(1)) AS count(*) AS point_count |
|
||||
| | Filter: count(Int64(1)) > Int64(1) |
|
||||
| | Aggregate: groupBy=[[date_trunc(Utf8("minute"), tql_grouped.ts)]], aggr=[[count(tql_grouped.ts) AS count(Int64(1))]] |
|
||||
| | SubqueryAlias: tql_grouped |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels, partial_filters=[labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with UNION
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
@@ -575,35 +574,35 @@ SELECT 'host1' as source, ts, cpu FROM host1_data
|
||||
UNION ALL
|
||||
SELECT 'host2' as source, ts, cpu FROM host2_data;
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Union |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: Utf8("host1") AS source, host1_data.ts, host1_data.cpu |
|
||||
| | SubqueryAlias: host1_data |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host1") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: Utf8("host2") AS source, host2_data.ts, host2_data.cpu |
|
||||
| | SubqueryAlias: host2_data |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host2") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | ]] |
|
||||
| physical_plan | InterleaveExec |
|
||||
| | CooperativeExec |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Union |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: Utf8("host1") AS source, host1_data.ts, host1_data.cpu |
|
||||
| | SubqueryAlias: host1_data |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host1") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels, partial_filters=[labels.host = Utf8("host1"), labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: Utf8("host2") AS source, host2_data.ts, host2_data.cpu |
|
||||
| | SubqueryAlias: host2_data |
|
||||
| | Projection: labels.ts AS ts, labels.host AS host, labels.cpu AS cpu |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.host = Utf8("host2") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels, partial_filters=[labels.host = Utf8("host2"), labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | InterleaveExec |
|
||||
| | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | CooperativeExec |
|
||||
| | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- Nested CTEs with TQL
|
||||
WITH
|
||||
@@ -638,31 +637,30 @@ EXPLAIN WITH
|
||||
)
|
||||
SELECT count(*) as high_values FROM final;
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) AS high_values |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] |
|
||||
| | SubqueryAlias: final |
|
||||
| | Projection: processed.ts AS ts, processed.percent AS percent |
|
||||
| | Projection: processed.ts, processed.percent |
|
||||
| | Filter: processed.percent > Float64(200) |
|
||||
| | SubqueryAlias: processed |
|
||||
| | Projection: base_tql.ts AS ts, percent AS percent |
|
||||
| | Projection: base_tql.ts, base_tql.val * Float64(100) AS percent |
|
||||
| | Filter: base_tql.val > Float64(0) |
|
||||
| | SubqueryAlias: base_tql |
|
||||
| | Projection: metric.ts AS ts, metric.val AS val |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) AS high_values |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] |
|
||||
| | SubqueryAlias: final |
|
||||
| | Projection: processed.ts AS ts, processed.percent AS percent |
|
||||
| | Projection: processed.ts, processed.percent |
|
||||
| | SubqueryAlias: processed |
|
||||
| | Projection: base_tql.ts AS ts, percent AS percent |
|
||||
| | Projection: base_tql.ts, base_tql.val * Float64(100) AS percent |
|
||||
| | SubqueryAlias: base_tql |
|
||||
| | Projection: metric.ts AS ts, metric.val AS val |
|
||||
| | Filter: metric.val * Float64(100) > Float64(200) AND metric.val > Float64(0) |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-299999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with time-based functions
|
||||
WITH time_shifted AS (
|
||||
@@ -680,22 +678,22 @@ EXPLAIN WITH time_shifted AS (
|
||||
)
|
||||
SELECT * FROM time_shifted;
|
||||
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: time_shifted.ts, time_shifted.val |
|
||||
| | SubqueryAlias: time_shifted |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesNormalize: offset=[50000], time index=[ts], filter NaN: [false] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-349999, None) AND metric.ts <= TimestampMillisecond(-10000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: time_shifted.ts, time_shifted.val |
|
||||
| | SubqueryAlias: time_shifted |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesNormalize: offset=[50000], time index=[ts], filter NaN: [false] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-349999, None) AND metric.ts <= TimestampMillisecond(-10000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-349999, None), metric.ts <= TimestampMillisecond(-10000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+------------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- TQL CTE with JOIN between TQL and regular table
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
@@ -754,16 +752,16 @@ LIMIT 5;
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.ts >= TimestampMillisecond(-29999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | TableScan: labels, partial_filters=[labels.ts >= TimestampMillisecond(-29999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| | Filter: l.host = Utf8("host1") |
|
||||
| | Projection: l.ts, l.host |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: l.ts, l.host |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: l |
|
||||
| | TableScan: labels |
|
||||
| | Filter: labels.host = Utf8("host1") |
|
||||
| | TableScan: labels, partial_filters=[labels.host = Utf8("host1")] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], fetch=5 |
|
||||
| | SortExec: TopK(fetch=5), expr=[ts@0 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], preserve_REDACTED
|
||||
| physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST, host@2 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], fetch=5 |
|
||||
| | SortExec: TopK(fetch=5), expr=[ts@0 ASC NULLS LAST, host@2 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], preserve_REDACTED
|
||||
| | ProjectionExec: expr=[ts@0 as ts, cpu@1 as avg_value, host@2 as host] |
|
||||
| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(date_trunc(Utf8("second"),t.ts)@2, date_trunc(Utf8("second"),l.ts)@2)], projection=[ts@0, cpu@1, host@4] |
|
||||
| | RepartitionExec: REDACTED
|
||||
@@ -771,9 +769,7 @@ LIMIT 5;
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: REDACTED
|
||||
| | ProjectionExec: expr=[ts@0 as ts, host@1 as host, date_trunc(second, ts@0) as date_trunc(Utf8("second"),l.ts)] |
|
||||
| | FilterExec: host@1 = host1 |
|
||||
| | ProjectionExec: expr=[ts@0 as ts, host@1 as host] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
@@ -834,16 +830,16 @@ LIMIT 5;
|
||||
| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] |
|
||||
| | PromSeriesDivide: tags=["host"] |
|
||||
| | Filter: labels.ts >= TimestampMillisecond(-29999, None) AND labels.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: labels |
|
||||
| | TableScan: labels, partial_filters=[labels.ts >= TimestampMillisecond(-29999, None), labels.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| | Filter: l.host = Utf8("host1") |
|
||||
| | Projection: l.ts, l.host |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: l.ts, l.host |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: l |
|
||||
| | TableScan: labels |
|
||||
| | Filter: labels.host = Utf8("host1") |
|
||||
| | TableScan: labels, partial_filters=[labels.host = Utf8("host1")] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], fetch=5 |
|
||||
| | SortExec: TopK(fetch=5), expr=[ts@0 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], preserve_REDACTED
|
||||
| physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST, host@2 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], fetch=5 |
|
||||
| | SortExec: TopK(fetch=5), expr=[ts@0 ASC NULLS LAST, host@2 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], preserve_REDACTED
|
||||
| | ProjectionExec: expr=[ts@1 as ts, cpu@0 as avg_value, host@2 as host] |
|
||||
| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(date_trunc(Utf8("second"),t.ts)@2, date_trunc(Utf8("second"),l.ts)@2)], projection=[cpu@0, ts@1, host@4] |
|
||||
| | RepartitionExec: REDACTED
|
||||
@@ -851,9 +847,7 @@ LIMIT 5;
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: REDACTED
|
||||
| | ProjectionExec: expr=[ts@0 as ts, host@1 as host, date_trunc(second, ts@0) as date_trunc(Utf8("second"),l.ts)] |
|
||||
| | FilterExec: host@1 = host1 |
|
||||
| | ProjectionExec: expr=[ts@0 as ts, host@1 as host] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
@@ -910,22 +904,22 @@ EXPLAIN WITH tql_lookback AS (
|
||||
)
|
||||
SELECT count(*) FROM tql_lookback;
|
||||
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(tql_lookback.ts) AS count(Int64(1))]] |
|
||||
| | SubqueryAlias: tql_lookback |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[15000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-14999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: count(Int64(1)) AS count(*) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[count(tql_lookback.ts) AS count(Int64(1))]] |
|
||||
| | SubqueryAlias: tql_lookback |
|
||||
| | PromInstantManipulate: range=[0..40000], lookback=[15000], interval=[10000], time index=[ts] |
|
||||
| | PromSeriesDivide: tags=[] |
|
||||
| | Filter: metric.ts >= TimestampMillisecond(-14999, None) AND metric.ts <= TimestampMillisecond(40000, None) |
|
||||
| | TableScan: metric, partial_filters=[metric.ts >= TimestampMillisecond(-14999, None), metric.ts <= TimestampMillisecond(40000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| | |
|
||||
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
drop table metric;
|
||||
|
||||
|
||||
@@ -22,20 +22,13 @@ EXPLAIN SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) LIMIT
|
||||
|
||||
EXPLAIN SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) WHERE 1=0;
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: sum(numbers.number) |
|
||||
| | Filter: Boolean(false) |
|
||||
| | Limit: skip=0, fetch=100000000000 |
|
||||
| | Projection: sum(numbers.number) |
|
||||
| | Aggregate: groupBy=[[]], aggr=[[sum(CAST(numbers.number AS UInt64))]] |
|
||||
| | TableScan: numbers |
|
||||
| | ]] |
|
||||
| physical_plan | EmptyExec |
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------+
|
||||
+---------------+-----------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------+
|
||||
| logical_plan | EmptyRelation: rows=0 |
|
||||
| physical_plan | EmptyExec |
|
||||
| | |
|
||||
+---------------+-----------------------+
|
||||
|
||||
CREATE TABLE test (a TIMESTAMP TIME INDEX, b INTEGER);
|
||||
|
||||
|
||||
138
tests/cases/standalone/optimizer/join_filter_pushdown.result
Normal file
138
tests/cases/standalone/optimizer/join_filter_pushdown.result
Normal file
@@ -0,0 +1,138 @@
|
||||
-- Regression test for issue #8338: static side-local predicates on a JOIN input
|
||||
-- should reach the remote region TableScan/SeqScan filters before MergeScan wrapping.
|
||||
CREATE TABLE parent (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
v DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE TABLE child (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
INSERT INTO parent VALUES
|
||||
('2024-01-30 00:00:00', 'a', 1.0),
|
||||
('2024-01-30 01:00:00', 'b', 2.0);
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
INSERT INTO child VALUES
|
||||
('2024-01-30 00:00:00', 'a'),
|
||||
('2024-01-31 00:00:00', 'c');
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
ADMIN FLUSH_TABLE('parent');
|
||||
|
||||
+-----------------------------+
|
||||
| ADMIN FLUSH_TABLE('parent') |
|
||||
+-----------------------------+
|
||||
| 0 |
|
||||
+-----------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('child');
|
||||
|
||||
+----------------------------+
|
||||
| ADMIN FLUSH_TABLE('child') |
|
||||
+----------------------------+
|
||||
| 0 |
|
||||
+----------------------------+
|
||||
|
||||
-- Query A: single table with time index filter (baseline).
|
||||
-- The scan should have partial_filters with the time condition.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE peers=\[\d+\(\d+,\s+\d+\),\s\] peers=[REDACTED]
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM parent WHERE ts >= '2024-01-30 00:00:00';
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: parent.ts, parent.k, parent.v |
|
||||
| | Filter: parent.ts >= TimestampMillisecond(1706572800000, None) |
|
||||
| | TableScan: parent, partial_filters=[parent.ts >= TimestampMillisecond(1706572800000, None)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: peers=[REDACTED] |
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- Query B: flat LEFT JOIN with WHERE on parent's ts.
|
||||
-- The filter should be pushed into the left MergeScan remote_input,
|
||||
-- appearing as partial_filters=[...] in the parent TableScan.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE peers=\[\d+\(\d+,\s+\d+\),\s\] peers=[REDACTED]
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM parent p LEFT JOIN child c ON p.k = c.k WHERE p.ts >= '2024-01-30 00:00:00';
|
||||
|
||||
+---------------+-------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Left Join: p.k = c.k |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: p |
|
||||
| | Filter: parent.ts >= TimestampMillisecond(1706572800000, None) |
|
||||
| | TableScan: parent, partial_filters=[parent.ts >= TimestampMillisecond(1706572800000, None)] |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | Filter: child.k IS NOT NULL |
|
||||
| | TableScan: child, partial_filters=[child.k IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=Left, on=[(k@1, k@1)] |
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | MergeScanExec: peers=[REDACTED] |
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | MergeScanExec: peers=[REDACTED] |
|
||||
| | |
|
||||
+---------------+-------------------------------------------------------------------------------------------------+
|
||||
|
||||
-- Query C: subquery pre-filter (workaround from #8338).
|
||||
-- Should produce the same logical scan filter shape as query B.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE peers=\[\d+\(\d+,\s+\d+\),\s\] peers=[REDACTED]
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM (SELECT * FROM parent WHERE ts >= '2024-01-30 00:00:00') p LEFT JOIN child c ON p.k = c.k;
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Left Join: p.k = c.k |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: p |
|
||||
| | Projection: parent.ts, parent.k, parent.v |
|
||||
| | Filter: parent.ts >= TimestampMillisecond(1706572800000, None) |
|
||||
| | TableScan: parent, partial_filters=[parent.ts >= TimestampMillisecond(1706572800000, None)] |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: c |
|
||||
| | Filter: child.k IS NOT NULL |
|
||||
| | TableScan: child, partial_filters=[child.k IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=Left, on=[(k@1, k@1)] |
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | MergeScanExec: peers=[REDACTED] |
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | MergeScanExec: peers=[REDACTED] |
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------+
|
||||
|
||||
DROP TABLE parent;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE child;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
54
tests/cases/standalone/optimizer/join_filter_pushdown.sql
Normal file
54
tests/cases/standalone/optimizer/join_filter_pushdown.sql
Normal file
@@ -0,0 +1,54 @@
|
||||
-- Regression test for issue #8338: static side-local predicates on a JOIN input
|
||||
-- should reach the remote region TableScan/SeqScan filters before MergeScan wrapping.
|
||||
|
||||
CREATE TABLE parent (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
v DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
CREATE TABLE child (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
INSERT INTO parent VALUES
|
||||
('2024-01-30 00:00:00', 'a', 1.0),
|
||||
('2024-01-30 01:00:00', 'b', 2.0);
|
||||
|
||||
INSERT INTO child VALUES
|
||||
('2024-01-30 00:00:00', 'a'),
|
||||
('2024-01-31 00:00:00', 'c');
|
||||
|
||||
ADMIN FLUSH_TABLE('parent');
|
||||
ADMIN FLUSH_TABLE('child');
|
||||
|
||||
-- Query A: single table with time index filter (baseline).
|
||||
-- The scan should have partial_filters with the time condition.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE peers=\[\d+\(\d+,\s+\d+\),\s\] peers=[REDACTED]
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM parent WHERE ts >= '2024-01-30 00:00:00';
|
||||
|
||||
-- Query B: flat LEFT JOIN with WHERE on parent's ts.
|
||||
-- The filter should be pushed into the left MergeScan remote_input,
|
||||
-- appearing as partial_filters=[...] in the parent TableScan.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE peers=\[\d+\(\d+,\s+\d+\),\s\] peers=[REDACTED]
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM parent p LEFT JOIN child c ON p.k = c.k WHERE p.ts >= '2024-01-30 00:00:00';
|
||||
|
||||
-- Query C: subquery pre-filter (workaround from #8338).
|
||||
-- Should produce the same logical scan filter shape as query B.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE peers=\[\d+\(\d+,\s+\d+\),\s\] peers=[REDACTED]
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM (SELECT * FROM parent WHERE ts >= '2024-01-30 00:00:00') p LEFT JOIN child c ON p.k = c.k;
|
||||
|
||||
DROP TABLE parent;
|
||||
DROP TABLE child;
|
||||
@@ -0,0 +1,285 @@
|
||||
-- Edge-case coverage for the selected pre-MergeScan optimizer prepass.
|
||||
-- These cases focus on keeping distributed pushdown safe around subqueries,
|
||||
-- nullable join keys, deterministic expressions, and nested predicates.
|
||||
CREATE TABLE fact (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
val DOUBLE,
|
||||
`region` STRING,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE TABLE dim (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
label STRING,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE TABLE vals (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
x INT,
|
||||
y INT
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
INSERT INTO fact VALUES
|
||||
('2024-01-30 00:00:00', 'a', 1.0, 'us'),
|
||||
('2024-01-30 01:00:00', 'b', 2.0, 'eu'),
|
||||
('2024-01-30 02:00:00', 'c', 3.0, 'us'),
|
||||
('2024-01-30 03:00:00', 'd', 0.5, 'eu');
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
INSERT INTO dim VALUES
|
||||
('2024-01-30 00:00:00', 'a', 'label_a'),
|
||||
('2024-01-31 00:00:00', 'c', 'label_c'),
|
||||
('2024-02-01 00:00:00', NULL, 'label_null'),
|
||||
('2024-02-02 00:00:00', 'e', 'label_e');
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
INSERT INTO vals VALUES
|
||||
('2024-01-30 00:00:00', 1, 10),
|
||||
('2024-01-30 01:00:00', 2, 20),
|
||||
('2024-01-30 02:00:00', 3, NULL),
|
||||
('2024-01-30 03:00:00', NULL, 40);
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
ADMIN FLUSH_TABLE('fact');
|
||||
|
||||
+---------------------------+
|
||||
| ADMIN FLUSH_TABLE('fact') |
|
||||
+---------------------------+
|
||||
| 0 |
|
||||
+---------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('dim');
|
||||
|
||||
+--------------------------+
|
||||
| ADMIN FLUSH_TABLE('dim') |
|
||||
+--------------------------+
|
||||
| 0 |
|
||||
+--------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('vals');
|
||||
|
||||
+---------------------------+
|
||||
| ADMIN FLUSH_TABLE('vals') |
|
||||
+---------------------------+
|
||||
| 0 |
|
||||
+---------------------------+
|
||||
|
||||
-- Correlated EXISTS plus LEFT JOIN: static fact-side filters should still be
|
||||
-- pushed into the fact scan, while the correlated predicate is decorrelated
|
||||
-- before PushDownFilter so no raw subquery or outer reference reaches
|
||||
-- partial_filters.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT f.k, f.val FROM fact f
|
||||
LEFT JOIN dim d ON f.k = d.k
|
||||
WHERE f.ts >= '2024-01-30 00:00:00'
|
||||
AND f.val > 1.0
|
||||
AND EXISTS (SELECT 1 FROM vals v WHERE v.x = CAST(f.val AS INT));
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | LeftSemi Join: CAST(f.val AS Int32) = __correlated_sq_1.x |
|
||||
| | Projection: f.k, f.val |
|
||||
| | Left Join: f.k = d.k |
|
||||
| | Projection: f.k, f.val |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: f |
|
||||
| | Filter: CAST(fact.val AS Int32) IS NOT NULL AND fact.ts >= TimestampMillisecond(1706572800000, None) AND fact.val > Float64(1) |
|
||||
| | TableScan: fact, partial_filters=[fact.ts >= TimestampMillisecond(1706572800000, None), fact.val > Float64(1), CAST(fact.val AS Int32) IS NOT NULL] |
|
||||
| | ]] |
|
||||
| | Projection: d.k |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: d |
|
||||
| | Filter: dim.k IS NOT NULL |
|
||||
| | TableScan: dim, partial_filters=[dim.k IS NOT NULL] |
|
||||
| | ]] |
|
||||
| | Projection: __correlated_sq_1.x |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: __correlated_sq_1 |
|
||||
| | Projection: Int64(1), v.x |
|
||||
| | SubqueryAlias: v |
|
||||
| | Filter: vals.x IS NOT NULL |
|
||||
| | TableScan: vals, partial_filters=[vals.x IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(CAST(f.val AS Int32)@2, x@0)], projection=[k@0, val@1] |
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | ProjectionExec: expr=[k@0 as k, val@1 as val, CAST(val@1 AS Int32) as CAST(f.val AS Int32)] |
|
||||
| | HashJoinExec: mode=Partitioned, join_type=Left, on=[(k@0, k@0)], projection=[k@0, val@1] |
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | ProjectionExec: expr=[k@1 as k, val@2 as val] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | ProjectionExec: expr=[k@1 as k] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | ProjectionExec: expr=[x@1 as x] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
SELECT f.k, f.val, d.label FROM fact f
|
||||
LEFT JOIN dim d ON f.k = d.k
|
||||
WHERE f.ts >= '2024-01-30 00:00:00'
|
||||
AND f.val > 1.0
|
||||
AND EXISTS (SELECT 1 FROM vals v WHERE v.x = CAST(f.val AS INT))
|
||||
ORDER BY f.k;
|
||||
|
||||
+---+-----+---------+
|
||||
| k | val | label |
|
||||
+---+-----+---------+
|
||||
| b | 2.0 | |
|
||||
| c | 3.0 | label_c |
|
||||
+---+-----+---------+
|
||||
|
||||
-- Semi/anti joins with nullable build-side keys. NULL keys from dim must not
|
||||
-- produce matches, and must not suppress NOT EXISTS results.
|
||||
SELECT f.k, f.val FROM fact f
|
||||
WHERE f.k IN (SELECT k FROM dim)
|
||||
ORDER BY f.k;
|
||||
|
||||
+---+-----+
|
||||
| k | val |
|
||||
+---+-----+
|
||||
| a | 1.0 |
|
||||
| c | 3.0 |
|
||||
+---+-----+
|
||||
|
||||
SELECT f.k, f.val FROM fact f
|
||||
WHERE EXISTS (SELECT 1 FROM dim d WHERE d.k = f.k)
|
||||
ORDER BY f.k;
|
||||
|
||||
+---+-----+
|
||||
| k | val |
|
||||
+---+-----+
|
||||
| a | 1.0 |
|
||||
| c | 3.0 |
|
||||
+---+-----+
|
||||
|
||||
SELECT f.k, f.val FROM fact f
|
||||
WHERE NOT EXISTS (SELECT 1 FROM dim d WHERE d.k = f.k)
|
||||
ORDER BY f.k;
|
||||
|
||||
+---+-----+
|
||||
| k | val |
|
||||
+---+-----+
|
||||
| b | 2.0 |
|
||||
| d | 0.5 |
|
||||
+---+-----+
|
||||
|
||||
-- Deterministic expression pushdown: casts may be pushed, but only as scan-local
|
||||
-- expressions that the remote scan can evaluate.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM fact WHERE CAST(fact.val AS INT) > 0;
|
||||
|
||||
+---------------+---------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: fact.ts, fact.k, fact.val, fact.region |
|
||||
| | Filter: CAST(fact.val AS Int32) > Int32(0) |
|
||||
| | TableScan: fact, partial_filters=[CAST(fact.val AS Int32) > Int32(0)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------+
|
||||
|
||||
CREATE TABLE edge_events (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
host STRING,
|
||||
rack INT,
|
||||
cpu DOUBLE,
|
||||
PRIMARY KEY (host)
|
||||
)
|
||||
PARTITION ON COLUMNS (rack) (
|
||||
rack < 10,
|
||||
rack >= 10 AND rack < 20,
|
||||
rack >= 20
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
INSERT INTO edge_events VALUES
|
||||
('2024-01-30 00:00:00', 'h1', 5, 0.5),
|
||||
('2024-01-30 01:00:00', 'h2', 15, 0.8),
|
||||
('2024-01-30 02:00:00', 'h3', 25, 0.3),
|
||||
('2024-01-30 03:00:00', 'h4', 25, 0.1);
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
ADMIN FLUSH_TABLE('edge_events');
|
||||
|
||||
+----------------------------------+
|
||||
| ADMIN FLUSH_TABLE('edge_events') |
|
||||
+----------------------------------+
|
||||
| 0 |
|
||||
+----------------------------------+
|
||||
|
||||
-- Nested OR/AND over a partition column plus non-partition predicates. This
|
||||
-- covers filter extraction without treating conjunctive scan filters as
|
||||
-- independent top-level pruning predicates.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM edge_events
|
||||
WHERE (rack > 5 AND cpu > 0.4) OR (host = 'h3');
|
||||
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: edge_events.ts, edge_events.host, edge_events.rack, edge_events.cpu |
|
||||
| | Filter: edge_events.rack > Int32(5) AND edge_events.cpu > Float64(0.4) OR edge_events.host = Utf8("h3") |
|
||||
| | TableScan: edge_events, partial_filters=[edge_events.rack > Int32(5) AND edge_events.cpu > Float64(0.4) OR edge_events.host = Utf8("h3")] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
SELECT host, rack, cpu FROM edge_events
|
||||
WHERE (rack > 5 AND cpu > 0.4) OR (host = 'h3')
|
||||
ORDER BY host;
|
||||
|
||||
+------+------+-----+
|
||||
| host | rack | cpu |
|
||||
+------+------+-----+
|
||||
| h2 | 15 | 0.8 |
|
||||
| h3 | 25 | 0.3 |
|
||||
+------+------+-----+
|
||||
|
||||
DROP TABLE fact;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE dim;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE vals;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE edge_events;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
129
tests/cases/standalone/optimizer/join_filter_pushdown_edge.sql
Normal file
129
tests/cases/standalone/optimizer/join_filter_pushdown_edge.sql
Normal file
@@ -0,0 +1,129 @@
|
||||
-- Edge-case coverage for the selected pre-MergeScan optimizer prepass.
|
||||
-- These cases focus on keeping distributed pushdown safe around subqueries,
|
||||
-- nullable join keys, deterministic expressions, and nested predicates.
|
||||
|
||||
CREATE TABLE fact (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
val DOUBLE,
|
||||
`region` STRING,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
CREATE TABLE dim (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
label STRING,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
CREATE TABLE vals (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
x INT,
|
||||
y INT
|
||||
) ENGINE = mito;
|
||||
|
||||
INSERT INTO fact VALUES
|
||||
('2024-01-30 00:00:00', 'a', 1.0, 'us'),
|
||||
('2024-01-30 01:00:00', 'b', 2.0, 'eu'),
|
||||
('2024-01-30 02:00:00', 'c', 3.0, 'us'),
|
||||
('2024-01-30 03:00:00', 'd', 0.5, 'eu');
|
||||
|
||||
INSERT INTO dim VALUES
|
||||
('2024-01-30 00:00:00', 'a', 'label_a'),
|
||||
('2024-01-31 00:00:00', 'c', 'label_c'),
|
||||
('2024-02-01 00:00:00', NULL, 'label_null'),
|
||||
('2024-02-02 00:00:00', 'e', 'label_e');
|
||||
|
||||
INSERT INTO vals VALUES
|
||||
('2024-01-30 00:00:00', 1, 10),
|
||||
('2024-01-30 01:00:00', 2, 20),
|
||||
('2024-01-30 02:00:00', 3, NULL),
|
||||
('2024-01-30 03:00:00', NULL, 40);
|
||||
|
||||
ADMIN FLUSH_TABLE('fact');
|
||||
ADMIN FLUSH_TABLE('dim');
|
||||
ADMIN FLUSH_TABLE('vals');
|
||||
|
||||
-- Correlated EXISTS plus LEFT JOIN: static fact-side filters should still be
|
||||
-- pushed into the fact scan, while the correlated predicate is decorrelated
|
||||
-- before PushDownFilter so no raw subquery or outer reference reaches
|
||||
-- partial_filters.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT f.k, f.val FROM fact f
|
||||
LEFT JOIN dim d ON f.k = d.k
|
||||
WHERE f.ts >= '2024-01-30 00:00:00'
|
||||
AND f.val > 1.0
|
||||
AND EXISTS (SELECT 1 FROM vals v WHERE v.x = CAST(f.val AS INT));
|
||||
|
||||
SELECT f.k, f.val, d.label FROM fact f
|
||||
LEFT JOIN dim d ON f.k = d.k
|
||||
WHERE f.ts >= '2024-01-30 00:00:00'
|
||||
AND f.val > 1.0
|
||||
AND EXISTS (SELECT 1 FROM vals v WHERE v.x = CAST(f.val AS INT))
|
||||
ORDER BY f.k;
|
||||
|
||||
-- Semi/anti joins with nullable build-side keys. NULL keys from dim must not
|
||||
-- produce matches, and must not suppress NOT EXISTS results.
|
||||
SELECT f.k, f.val FROM fact f
|
||||
WHERE f.k IN (SELECT k FROM dim)
|
||||
ORDER BY f.k;
|
||||
|
||||
SELECT f.k, f.val FROM fact f
|
||||
WHERE EXISTS (SELECT 1 FROM dim d WHERE d.k = f.k)
|
||||
ORDER BY f.k;
|
||||
|
||||
SELECT f.k, f.val FROM fact f
|
||||
WHERE NOT EXISTS (SELECT 1 FROM dim d WHERE d.k = f.k)
|
||||
ORDER BY f.k;
|
||||
|
||||
-- Deterministic expression pushdown: casts may be pushed, but only as scan-local
|
||||
-- expressions that the remote scan can evaluate.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM fact WHERE CAST(fact.val AS INT) > 0;
|
||||
|
||||
CREATE TABLE edge_events (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
host STRING,
|
||||
rack INT,
|
||||
cpu DOUBLE,
|
||||
PRIMARY KEY (host)
|
||||
)
|
||||
PARTITION ON COLUMNS (rack) (
|
||||
rack < 10,
|
||||
rack >= 10 AND rack < 20,
|
||||
rack >= 20
|
||||
) ENGINE = mito;
|
||||
|
||||
INSERT INTO edge_events VALUES
|
||||
('2024-01-30 00:00:00', 'h1', 5, 0.5),
|
||||
('2024-01-30 01:00:00', 'h2', 15, 0.8),
|
||||
('2024-01-30 02:00:00', 'h3', 25, 0.3),
|
||||
('2024-01-30 03:00:00', 'h4', 25, 0.1);
|
||||
|
||||
ADMIN FLUSH_TABLE('edge_events');
|
||||
|
||||
-- Nested OR/AND over a partition column plus non-partition predicates. This
|
||||
-- covers filter extraction without treating conjunctive scan filters as
|
||||
-- independent top-level pruning predicates.
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT * FROM edge_events
|
||||
WHERE (rack > 5 AND cpu > 0.4) OR (host = 'h3');
|
||||
|
||||
SELECT host, rack, cpu FROM edge_events
|
||||
WHERE (rack > 5 AND cpu > 0.4) OR (host = 'h3')
|
||||
ORDER BY host;
|
||||
|
||||
DROP TABLE fact;
|
||||
DROP TABLE dim;
|
||||
DROP TABLE vals;
|
||||
DROP TABLE edge_events;
|
||||
93
tests/cases/standalone/optimizer/lateral_join_guard.result
Normal file
93
tests/cases/standalone/optimizer/lateral_join_guard.result
Normal file
@@ -0,0 +1,93 @@
|
||||
-- Document the current aliased SQL LATERAL limitation and guard the remote
|
||||
-- scan boundary. DataFusion's DecorrelateLateralJoin does not currently match
|
||||
-- the SubqueryAlias(Subquery) shape produced by `LATERAL (...) d`, so this query
|
||||
-- is still expected to fail physical planning with an outer_ref expression. The
|
||||
-- important regression assertion is that the remaining outer_ref predicate must
|
||||
-- NOT be advertised as a remote TableScan.partial_filters predicate.
|
||||
CREATE TABLE lateral_fact (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
val DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE TABLE lateral_dim (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
threshold DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
INSERT INTO lateral_fact VALUES
|
||||
('2024-01-30 00:00:00', 'a', 10.0),
|
||||
('2024-01-30 01:00:00', 'b', 20.0);
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
INSERT INTO lateral_dim VALUES
|
||||
('2024-01-30 00:00:00', 'a', 5.0),
|
||||
('2024-01-30 01:00:00', 'b', 25.0);
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
ADMIN FLUSH_TABLE('lateral_fact');
|
||||
|
||||
+-----------------------------------+
|
||||
| ADMIN FLUSH_TABLE('lateral_fact') |
|
||||
+-----------------------------------+
|
||||
| 0 |
|
||||
+-----------------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('lateral_dim');
|
||||
|
||||
+----------------------------------+
|
||||
| ADMIN FLUSH_TABLE('lateral_dim') |
|
||||
+----------------------------------+
|
||||
| 0 |
|
||||
+----------------------------------+
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
EXPLAIN SELECT f.k, d.threshold
|
||||
FROM lateral_fact f,
|
||||
LATERAL (
|
||||
SELECT threshold FROM lateral_dim d WHERE d.k = f.k
|
||||
) d
|
||||
WHERE f.val > d.threshold
|
||||
ORDER BY f.k;
|
||||
|
||||
+---------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Sort: f.k ASC NULLS LAST |
|
||||
| | Projection: f.k, d.threshold |
|
||||
| | Inner Join: Filter: f.val > d.threshold |
|
||||
| | Projection: f.k, f.val |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: f |
|
||||
| | TableScan: lateral_fact |
|
||||
| | ]] |
|
||||
| | SubqueryAlias: d |
|
||||
| | Subquery: |
|
||||
| | SubqueryAlias: d |
|
||||
| | Projection: lateral_dim.threshold |
|
||||
| | Filter: lateral_dim.k = outer_ref(f.k) |
|
||||
| | Projection: lateral_dim.k, lateral_dim.threshold |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | TableScan: lateral_dim |
|
||||
| | ]] |
|
||||
| physical_plan_error | This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(Field { name: "k", data_type: Utf8, nullable: true }, Column { relation: Some(Bare { table: "f" }), name: "k" }) |
|
||||
+---------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
DROP TABLE lateral_fact;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE lateral_dim;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
44
tests/cases/standalone/optimizer/lateral_join_guard.sql
Normal file
44
tests/cases/standalone/optimizer/lateral_join_guard.sql
Normal file
@@ -0,0 +1,44 @@
|
||||
-- Document the current aliased SQL LATERAL limitation and guard the remote
|
||||
-- scan boundary. DataFusion's DecorrelateLateralJoin does not currently match
|
||||
-- the SubqueryAlias(Subquery) shape produced by `LATERAL (...) d`, so this query
|
||||
-- is still expected to fail physical planning with an outer_ref expression. The
|
||||
-- important regression assertion is that the remaining outer_ref predicate must
|
||||
-- NOT be advertised as a remote TableScan.partial_filters predicate.
|
||||
|
||||
CREATE TABLE lateral_fact (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
val DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
CREATE TABLE lateral_dim (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
threshold DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
INSERT INTO lateral_fact VALUES
|
||||
('2024-01-30 00:00:00', 'a', 10.0),
|
||||
('2024-01-30 01:00:00', 'b', 20.0);
|
||||
|
||||
INSERT INTO lateral_dim VALUES
|
||||
('2024-01-30 00:00:00', 'a', 5.0),
|
||||
('2024-01-30 01:00:00', 'b', 25.0);
|
||||
|
||||
ADMIN FLUSH_TABLE('lateral_fact');
|
||||
ADMIN FLUSH_TABLE('lateral_dim');
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
EXPLAIN SELECT f.k, d.threshold
|
||||
FROM lateral_fact f,
|
||||
LATERAL (
|
||||
SELECT threshold FROM lateral_dim d WHERE d.k = f.k
|
||||
) d
|
||||
WHERE f.val > d.threshold
|
||||
ORDER BY f.k;
|
||||
|
||||
DROP TABLE lateral_fact;
|
||||
DROP TABLE lateral_dim;
|
||||
158
tests/cases/standalone/optimizer/rewrite_set_comparison.result
Normal file
158
tests/cases/standalone/optimizer/rewrite_set_comparison.result
Normal file
@@ -0,0 +1,158 @@
|
||||
-- Set-comparison subqueries (`ANY`/`ALL`) must be rewritten before
|
||||
-- PushDownFilter. Otherwise the set-comparison subquery can be pushed into
|
||||
-- TableScan.partial_filters, which is not a valid remote scan filter.
|
||||
CREATE TABLE sc_t (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
v INT,
|
||||
PRIMARY KEY (v)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE TABLE sc_s (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
v INT
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
INSERT INTO sc_t VALUES
|
||||
('2024-01-30 00:00:00', 1),
|
||||
('2024-01-30 01:00:00', 6),
|
||||
('2024-01-30 02:00:00', 10);
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
INSERT INTO sc_s VALUES
|
||||
('2024-01-30 00:00:00', 5),
|
||||
('2024-01-30 01:00:00', NULL);
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
ADMIN FLUSH_TABLE('sc_t');
|
||||
|
||||
+---------------------------+
|
||||
| ADMIN FLUSH_TABLE('sc_t') |
|
||||
+---------------------------+
|
||||
| 0 |
|
||||
+---------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('sc_s');
|
||||
|
||||
+---------------------------+
|
||||
| ADMIN FLUSH_TABLE('sc_s') |
|
||||
+---------------------------+
|
||||
| 0 |
|
||||
+---------------------------+
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
EXPLAIN SELECT v FROM sc_t WHERE v > ANY(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Sort: sc_t.v ASC NULLS LAST |
|
||||
| | Projection: sc_t.v |
|
||||
| | Filter: __correlated_sq_1.mark OR __correlated_sq_2.mark AND NOT __correlated_sq_1.mark AND Boolean(NULL) |
|
||||
| | LeftMark Join: Filter: sc_t.v > __correlated_sq_2.v IS NULL |
|
||||
| | Filter: __correlated_sq_1.mark OR NOT __correlated_sq_1.mark AND Boolean(NULL) |
|
||||
| | LeftMark Join: Filter: sc_t.v > __correlated_sq_1.v IS TRUE |
|
||||
| | Projection: sc_t.v |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | TableScan: sc_t |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: __correlated_sq_1.v |
|
||||
| | SubqueryAlias: __correlated_sq_1 |
|
||||
| | Projection: sc_s.v |
|
||||
| | TableScan: sc_s |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: __correlated_sq_2.v |
|
||||
| | SubqueryAlias: __correlated_sq_2 |
|
||||
| | Projection: sc_s.v |
|
||||
| | TableScan: sc_s |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [v@0 ASC NULLS LAST] |
|
||||
| | SortExec: expr=[v@0 ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
| | FilterExec: mark@1 OR mark@2 AND NOT mark@1 AND NULL, projection=[v@0] |
|
||||
| | NestedLoopJoinExec: join_type=LeftMark, filter=v@0 > v@1 IS NULL |
|
||||
| | CoalescePartitionsExec |
|
||||
| | FilterExec: mark@1 OR NOT mark@1 AND NULL |
|
||||
| | NestedLoopJoinExec: join_type=LeftMark, filter=(v@0 > v@1) IS NOT DISTINCT FROM true |
|
||||
| | CoalescePartitionsExec |
|
||||
| | ProjectionExec: expr=[v@1 as v] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+---------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
SELECT v FROM sc_t WHERE v > ANY(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
+----+
|
||||
| v |
|
||||
+----+
|
||||
| 6 |
|
||||
| 10 |
|
||||
+----+
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
EXPLAIN SELECT v FROM sc_t WHERE v != ALL(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
+---------------+----------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+----------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Sort: sc_t.v ASC NULLS LAST |
|
||||
| | Projection: sc_t.v |
|
||||
| | Filter: __correlated_sq_2.mark AND Boolean(NULL) OR NOT __correlated_sq_2.mark |
|
||||
| | LeftMark Join: Filter: sc_t.v != __correlated_sq_2.v IS NULL |
|
||||
| | Projection: sc_t.v |
|
||||
| | Filter: NOT __correlated_sq_1.mark |
|
||||
| | LeftMark Join: Filter: sc_t.v != __correlated_sq_1.v IS FALSE |
|
||||
| | Projection: sc_t.v |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | TableScan: sc_t |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: __correlated_sq_1.v |
|
||||
| | SubqueryAlias: __correlated_sq_1 |
|
||||
| | Projection: sc_s.v |
|
||||
| | TableScan: sc_s |
|
||||
| | ]] |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: __correlated_sq_2.v |
|
||||
| | SubqueryAlias: __correlated_sq_2 |
|
||||
| | Projection: sc_s.v |
|
||||
| | TableScan: sc_s |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [v@0 ASC NULLS LAST] |
|
||||
| | SortExec: expr=[v@0 ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
| | FilterExec: mark@1 AND NULL OR NOT mark@1, projection=[v@0] |
|
||||
| | NestedLoopJoinExec: join_type=LeftMark, filter=v@0 != v@1 IS NULL |
|
||||
| | CoalescePartitionsExec |
|
||||
| | FilterExec: NOT mark@1, projection=[v@0] |
|
||||
| | NestedLoopJoinExec: join_type=LeftMark, filter=(v@0 != v@1) IS NOT DISTINCT FROM false |
|
||||
| | CoalescePartitionsExec |
|
||||
| | ProjectionExec: expr=[v@1 as v] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------+
|
||||
|
||||
SELECT v FROM sc_t WHERE v != ALL(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
++
|
||||
++
|
||||
|
||||
DROP TABLE sc_t;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE sc_s;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
41
tests/cases/standalone/optimizer/rewrite_set_comparison.sql
Normal file
41
tests/cases/standalone/optimizer/rewrite_set_comparison.sql
Normal file
@@ -0,0 +1,41 @@
|
||||
-- Set-comparison subqueries (`ANY`/`ALL`) must be rewritten before
|
||||
-- PushDownFilter. Otherwise the set-comparison subquery can be pushed into
|
||||
-- TableScan.partial_filters, which is not a valid remote scan filter.
|
||||
|
||||
CREATE TABLE sc_t (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
v INT,
|
||||
PRIMARY KEY (v)
|
||||
) ENGINE = mito;
|
||||
|
||||
CREATE TABLE sc_s (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
v INT
|
||||
) ENGINE = mito;
|
||||
|
||||
INSERT INTO sc_t VALUES
|
||||
('2024-01-30 00:00:00', 1),
|
||||
('2024-01-30 01:00:00', 6),
|
||||
('2024-01-30 02:00:00', 10);
|
||||
|
||||
INSERT INTO sc_s VALUES
|
||||
('2024-01-30 00:00:00', 5),
|
||||
('2024-01-30 01:00:00', NULL);
|
||||
|
||||
ADMIN FLUSH_TABLE('sc_t');
|
||||
ADMIN FLUSH_TABLE('sc_s');
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
EXPLAIN SELECT v FROM sc_t WHERE v > ANY(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
SELECT v FROM sc_t WHERE v > ANY(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
EXPLAIN SELECT v FROM sc_t WHERE v != ALL(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
SELECT v FROM sc_t WHERE v != ALL(SELECT v FROM sc_s) ORDER BY v;
|
||||
|
||||
DROP TABLE sc_t;
|
||||
DROP TABLE sc_s;
|
||||
114
tests/cases/standalone/optimizer/scalar_subquery_prepass.result
Normal file
114
tests/cases/standalone/optimizer/scalar_subquery_prepass.result
Normal file
@@ -0,0 +1,114 @@
|
||||
-- Scalar subquery predicates must be converted before PushDownFilter. Otherwise
|
||||
-- a scalar subquery can be pushed into TableScan.partial_filters, which is not a
|
||||
-- valid remote scan filter.
|
||||
CREATE TABLE scalar_fact (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
val DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE TABLE scalar_dim (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
threshold DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
INSERT INTO scalar_fact VALUES
|
||||
('2024-01-30 00:00:00', 'a', 10.0),
|
||||
('2024-01-30 01:00:00', 'b', 20.0),
|
||||
('2024-01-30 02:00:00', 'c', 30.0),
|
||||
('2024-01-30 03:00:00', 'd', 40.0);
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
INSERT INTO scalar_dim VALUES
|
||||
('2024-01-30 00:00:00', 'a', 5.0),
|
||||
('2024-01-30 01:00:00', 'b', 25.0),
|
||||
('2024-01-30 02:00:00', 'c', NULL);
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
ADMIN FLUSH_TABLE('scalar_fact');
|
||||
|
||||
+----------------------------------+
|
||||
| ADMIN FLUSH_TABLE('scalar_fact') |
|
||||
+----------------------------------+
|
||||
| 0 |
|
||||
+----------------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('scalar_dim');
|
||||
|
||||
+---------------------------------+
|
||||
| ADMIN FLUSH_TABLE('scalar_dim') |
|
||||
+---------------------------------+
|
||||
| 0 |
|
||||
+---------------------------------+
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT f.k, f.val FROM scalar_fact f
|
||||
WHERE f.val > (
|
||||
SELECT max(d.threshold) FROM scalar_dim d WHERE d.k = f.k
|
||||
)
|
||||
ORDER BY f.k;
|
||||
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| plan_type | plan |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------+
|
||||
| logical_plan | Sort: f.k ASC NULLS LAST |
|
||||
| | Projection: f.k, f.val |
|
||||
| | Inner Join: f.k = __scalar_sq_1.k Filter: f.val > __scalar_sq_1.max(d.threshold) |
|
||||
| | Projection: f.k, f.val |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: f |
|
||||
| | TableScan: scalar_fact |
|
||||
| | ]] |
|
||||
| | Projection: __scalar_sq_1.max(d.threshold), __scalar_sq_1.k |
|
||||
| | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | SubqueryAlias: __scalar_sq_1 |
|
||||
| | Projection: max(d.threshold), d.k, __always_true |
|
||||
| | Aggregate: groupBy=[[d.k, Boolean(true) AS __always_true]], aggr=[[max(d.threshold)]] |
|
||||
| | SubqueryAlias: d |
|
||||
| | Filter: scalar_dim.k IS NOT NULL |
|
||||
| | TableScan: scalar_dim, partial_filters=[scalar_dim.k IS NOT NULL] |
|
||||
| | ]] |
|
||||
| physical_plan | SortPreservingMergeExec: [k@0 ASC NULLS LAST] |
|
||||
| | SortExec: expr=[k@0 ASC NULLS LAST], preserve_partitioning=[true] |
|
||||
| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(k@0, k@1)], filter=val@0 > max(d.threshold)@1, projection=[k@0, val@1] |
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | ProjectionExec: expr=[k@1 as k, val@2 as val] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | RepartitionExec: partitioning=Hash([REDACTED
|
||||
| | ProjectionExec: expr=[max(d.threshold)@0 as max(d.threshold), k@1 as k] |
|
||||
| | MergeScanExec: REDACTED
|
||||
| | |
|
||||
+---------------+----------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
SELECT f.k, f.val FROM scalar_fact f
|
||||
WHERE f.val > (
|
||||
SELECT max(d.threshold) FROM scalar_dim d WHERE d.k = f.k
|
||||
)
|
||||
ORDER BY f.k;
|
||||
|
||||
+---+------+
|
||||
| k | val |
|
||||
+---+------+
|
||||
| a | 10.0 |
|
||||
+---+------+
|
||||
|
||||
DROP TABLE scalar_fact;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE scalar_dim;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
50
tests/cases/standalone/optimizer/scalar_subquery_prepass.sql
Normal file
50
tests/cases/standalone/optimizer/scalar_subquery_prepass.sql
Normal file
@@ -0,0 +1,50 @@
|
||||
-- Scalar subquery predicates must be converted before PushDownFilter. Otherwise
|
||||
-- a scalar subquery can be pushed into TableScan.partial_filters, which is not a
|
||||
-- valid remote scan filter.
|
||||
|
||||
CREATE TABLE scalar_fact (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
val DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
CREATE TABLE scalar_dim (
|
||||
ts TIMESTAMP(3) TIME INDEX,
|
||||
k STRING,
|
||||
threshold DOUBLE,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE = mito;
|
||||
|
||||
INSERT INTO scalar_fact VALUES
|
||||
('2024-01-30 00:00:00', 'a', 10.0),
|
||||
('2024-01-30 01:00:00', 'b', 20.0),
|
||||
('2024-01-30 02:00:00', 'c', 30.0),
|
||||
('2024-01-30 03:00:00', 'd', 40.0);
|
||||
|
||||
INSERT INTO scalar_dim VALUES
|
||||
('2024-01-30 00:00:00', 'a', 5.0),
|
||||
('2024-01-30 01:00:00', 'b', 25.0),
|
||||
('2024-01-30 02:00:00', 'c', NULL);
|
||||
|
||||
ADMIN FLUSH_TABLE('scalar_fact');
|
||||
ADMIN FLUSH_TABLE('scalar_dim');
|
||||
|
||||
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
||||
-- SQLNESS REPLACE (peers.*) REDACTED
|
||||
-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED
|
||||
-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED
|
||||
EXPLAIN SELECT f.k, f.val FROM scalar_fact f
|
||||
WHERE f.val > (
|
||||
SELECT max(d.threshold) FROM scalar_dim d WHERE d.k = f.k
|
||||
)
|
||||
ORDER BY f.k;
|
||||
|
||||
SELECT f.k, f.val FROM scalar_fact f
|
||||
WHERE f.val > (
|
||||
SELECT max(d.threshold) FROM scalar_dim d WHERE d.k = f.k
|
||||
)
|
||||
ORDER BY f.k;
|
||||
|
||||
DROP TABLE scalar_fact;
|
||||
DROP TABLE scalar_dim;
|
||||
@@ -67,7 +67,7 @@ WHERE
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: cpu.rack |
|
||||
| | Filter: cpu.usage_small = Int16(10) OR cpu.usage_small = Int16(20) |
|
||||
| | TableScan: cpu |
|
||||
| | TableScan: cpu, partial_filters=[cpu.usage_small = Int16(10) OR cpu.usage_small = Int16(20)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
@@ -88,7 +88,7 @@ WHERE
|
||||
| logical_plan | MergeScan [is_placeholder=false, remote_input=[ |
|
||||
| | Projection: cpu.rack |
|
||||
| | Filter: cpu.usage_small >= Int16(10) AND cpu.usage_small <= Int16(20) |
|
||||
| | TableScan: cpu |
|
||||
| | TableScan: cpu, partial_filters=[cpu.usage_small >= Int16(10), cpu.usage_small <= Int16(20)] |
|
||||
| | ]] |
|
||||
| physical_plan | CooperativeExec |
|
||||
| | MergeScanExec: REDACTED
|
||||
|
||||
Reference in New Issue
Block a user