From 4f5dccf6bb1a9174918a8641b78dddad028588d6 Mon Sep 17 00:00:00 2001 From: discord9 Date: Fri, 3 Jul 2026 17:10:00 +0800 Subject: [PATCH] fix(query): push down PromQL cast filters (#8407) * fix(query): push down PromQL cast filters Signed-off-by: discord9 * test(query): cover cast preimage pushdown Signed-off-by: discord9 * test(query): expand cast preimage coverage Signed-off-by: discord9 * test(query): compare cast normalization rules Signed-off-by: discord9 * test(query): update cast pushdown sqlness plans Signed-off-by: discord9 --------- Signed-off-by: discord9 --- Cargo.lock | 68 +-- Cargo.toml | 30 +- src/query/src/dist_plan/analyzer.rs | 2 + src/query/src/dist_plan/analyzer/test.rs | 84 ++++ src/query/src/dist_plan/commutativity.rs | 4 +- .../src/optimizer/const_normalization.rs | 134 ++++++ .../distributed/explain/subqueries.result | 2 +- .../common/filter/cast_preimage.result | 394 ++++++++++++++++++ .../common/filter/cast_preimage.sql | 175 ++++++++ .../common/promql/precisions.result | 136 ++++++ .../standalone/common/promql/precisions.sql | 60 +++ .../common/tql-explain-analyze/explain.result | 138 ++---- .../common/tql/general_table.result | 16 +- .../standalone/common/tql/general_table.sql | 4 + 14 files changed, 1091 insertions(+), 156 deletions(-) create mode 100644 tests/cases/standalone/common/filter/cast_preimage.result create mode 100644 tests/cases/standalone/common/filter/cast_preimage.sql diff --git a/Cargo.lock b/Cargo.lock index c563173084c..ed5c19ab2a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3644,7 +3644,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "arrow-schema 58.3.0", @@ -3698,7 +3698,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3722,7 +3722,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3744,7 +3744,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -3768,7 +3768,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "futures", "log", @@ -3778,7 +3778,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-compression", @@ -3812,7 +3812,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "arrow-ipc 58.3.0", @@ -3835,7 +3835,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3857,7 +3857,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3880,7 +3880,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3909,12 +3909,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" [[package]] name = "datafusion-execution" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -3936,7 +3936,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3958,7 +3958,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -3970,7 +3970,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -4001,7 +4001,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4022,7 +4022,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4034,7 +4034,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "arrow-ord 58.3.0", @@ -4058,7 +4058,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "async-trait", @@ -4073,7 +4073,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4090,7 +4090,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4099,7 +4099,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "datafusion-doc", "quote", @@ -4109,7 +4109,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "chrono", @@ -4158,7 +4158,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4181,7 +4181,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4195,7 +4195,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4211,7 +4211,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4229,7 +4229,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4260,7 +4260,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "chrono", @@ -4287,7 +4287,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4297,7 +4297,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4313,7 +4313,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "async-trait", "datafusion-common", @@ -4326,7 +4326,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "arrow 58.3.0", "bigdecimal 0.4.8", @@ -4344,7 +4344,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fe8422ed266dc8a527c06699057c20875a465173#fe8422ed266dc8a527c06699057c20875a465173" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 1197547bdc6..c7412cde0f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -341,21 +341,21 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git" rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" [patch.crates-io] -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fe8422ed266dc8a527c06699057c20875a465173" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x [profile.release] diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs index f98565e4f08..c41f7b618dc 100644 --- a/src/query/src/dist_plan/analyzer.rs +++ b/src/query/src/dist_plan/analyzer.rs @@ -36,6 +36,7 @@ use datafusion_optimizer::propagate_empty_relation::PropagateEmptyRelation; use datafusion_optimizer::push_down_filter::PushDownFilter; use datafusion_optimizer::rewrite_set_comparison::RewriteSetComparison; use datafusion_optimizer::scalar_subquery_to_join::ScalarSubqueryToJoin; +use datafusion_optimizer::simplify_expressions::SimplifyExpressions; use promql::extension_plan::SeriesDivide; use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; use table::metadata::TableType; @@ -219,6 +220,7 @@ fn pre_merge_scan_optimizer() -> Optimizer { Arc::new(PropagateEmptyRelation::new()), Arc::new(FilterNullJoinKeys::default()), Arc::new(PushDownFilter::new()), + Arc::new(SimplifyExpressions::new()), ]) } diff --git a/src/query/src/dist_plan/analyzer/test.rs b/src/query/src/dist_plan/analyzer/test.rs index b1417e213e4..04602d440a0 100644 --- a/src/query/src/dist_plan/analyzer/test.rs +++ b/src/query/src/dist_plan/analyzer/test.rs @@ -2263,6 +2263,90 @@ fn test_table_scan_projection() { assert_eq!(expected, result.to_string()); } +#[test] +fn test_table_scan_cast_projection_pushdown() { + init_default_ut_logging(); + let test_table = TestTable::table_with_name(0, "t".to_string()); + let table_provider = Arc::new(DfTableProviderAdapter::new(test_table)); + let table_source = Arc::new(DefaultTableSource::new(table_provider.clone() as _)); + let ctx = SessionContext::new(); + ctx.register_table(TableReference::bare("t"), table_provider.clone() as _) + .unwrap(); + + let scan = LogicalPlanBuilder::scan_with_filters("t", table_source, Some(vec![3]), vec![]) + .unwrap() + .build() + .unwrap(); + let ts_cast_type = DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())); + let ts_cast_expr = col("ts").cast_to(&ts_cast_type, scan.schema()).unwrap(); + let plan = LogicalPlanBuilder::from(scan) + .project(vec![ts_cast_expr.alias("ts")]) + .unwrap() + .build() + .unwrap(); + + let config = ConfigOptions::default(); + let result = DistPlannerAnalyzer {} + .analyze(plan.clone(), &config) + .unwrap(); + let expected = [ + "Projection: ts", + " MergeScan [is_placeholder=false, remote_input=[", + "Projection: CAST(t.ts AS Timestamp(ms, \"+00:00\")) AS ts", + " TableScan: t projection=[ts]", + "]]", + ] + .join("\n"); + assert_eq!(expected, result.to_string()); +} + +#[test] +fn test_cast_filter_simplified_after_pushdown() { + // This test invokes `DistPlannerAnalyzer` directly rather than the full + // query `SessionState`, so the globally-registered `ConstNormalizationRule` + // does not run here. The native timestamp filter below proves the focused + // pre-MergeScan pass can do this by running DataFusion `SimplifyExpressions` + // after `PushDownFilter` has pushed the alias predicate through the cast + // projection into `TableScan.filters`. + init_default_ut_logging(); + let test_table = + TestTable::table_with_filter_pushdown(0, "t".to_string(), FilterPushDownType::Inexact); + let table_provider = Arc::new(DfTableProviderAdapter::new(test_table)); + let table_source = Arc::new(DefaultTableSource::new(table_provider.clone() as _)); + let ctx = SessionContext::new(); + ctx.register_table(TableReference::bare("t"), table_provider.clone() as _) + .unwrap(); + + let scan = LogicalPlanBuilder::scan_with_filters("t", table_source, Some(vec![3]), vec![]) + .unwrap() + .build() + .unwrap(); + let ts_cast_type = DataType::Timestamp(TimeUnit::Second, None); + let ts_cast_expr = col("ts").cast_to(&ts_cast_type, scan.schema()).unwrap(); + let plan = LogicalPlanBuilder::from(scan) + .project(vec![ts_cast_expr.alias("ts")]) + .unwrap() + .filter(col("ts").gt_eq(lit(ScalarValue::TimestampSecond(Some(10), None)))) + .unwrap() + .build() + .unwrap(); + + let config = ConfigOptions::default(); + let result = DistPlannerAnalyzer {} + .analyze(plan.clone(), &config) + .unwrap(); + let expected = [ + "Projection: ts", + " MergeScan [is_placeholder=false, remote_input=[", + "Projection: CAST(t.ts AS Timestamp(s)) AS ts", + " Filter: t.ts >= TimestampMillisecond(10000, None)", + " TableScan: t projection=[ts], partial_filters=[t.ts >= TimestampMillisecond(10000, None)]", + "]]", + ] + .join("\n"); + assert_eq!(expected, result.to_string()); +} + /// Test that static side-local predicates on a JOIN input reach the remote /// region TableScan before MergeScan wrapping (issue #8338). /// diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs index 7366aa0d5b3..b380fbba05e 100644 --- a/src/query/src/dist_plan/commutativity.rs +++ b/src/query/src/dist_plan/commutativity.rs @@ -322,14 +322,14 @@ impl Categorizer { | Expr::SimilarTo(_) | Expr::IsUnknown(_) | Expr::IsNotUnknown(_) - | Expr::Cast(_) - | Expr::TryCast(_) | Expr::WindowFunction(_) | Expr::InSubquery(_) | Expr::ScalarSubquery(_) | Expr::Wildcard { .. } => Commutativity::Unimplemented, Expr::Alias(alias) => Self::check_expr(&alias.expr), + Expr::Cast(cast) => Self::check_expr(&cast.expr), + Expr::TryCast(try_cast) => Self::check_expr(&try_cast.expr), Expr::Unnest(_) | Expr::GroupingSet(_) diff --git a/src/query/src/optimizer/const_normalization.rs b/src/query/src/optimizer/const_normalization.rs index fac0e90538f..480a541b942 100644 --- a/src/query/src/optimizer/const_normalization.rs +++ b/src/query/src/optimizer/const_normalization.rs @@ -615,6 +615,7 @@ mod tests { use datafusion_optimizer::analyzer::AnalyzerRule; use datafusion_optimizer::optimizer::{Optimizer, OptimizerContext}; use datafusion_optimizer::push_down_filter::PushDownFilter; + use datafusion_optimizer::simplify_expressions::SimplifyExpressions; use table::predicate::build_time_range_predicate; use super::{ @@ -982,6 +983,127 @@ mod tests { ); } + #[test] + fn test_const_normalization_vs_datafusion_cast_preimage_overlap() { + struct Case { + name: &'static str, + fields: Vec, + predicate: Expr, + expected_greptime: &'static str, + expected_datafusion: &'static str, + } + + let cases = [ + Case { + name: "integer widening binary", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: cast(col("v"), DataType::Int64).gt_eq(lit(42_i64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(42)\n TableScan: t", + }, + Case { + name: "swapped integer comparison", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: lit(42_i64).lt_eq(cast(col("v"), DataType::Int64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(42)\n TableScan: t", + }, + Case { + name: "try_cast integer widening binary", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: try_cast(col("v"), DataType::Int64).gt_eq(lit(42_i64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(42)\n TableScan: t", + }, + Case { + name: "exact in-list", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: cast(col("v"), DataType::Int64) + .in_list(vec![lit(1_i64), lit(2_i64)], false), + expected_greptime: "Filter: t.v IN ([Int16(1), Int16(2)])\n TableScan: t", + expected_datafusion: "Filter: t.v = Int16(1) OR t.v = Int16(2)\n TableScan: t", + }, + Case { + name: "integer between", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: cast(col("v"), DataType::Int64).between(lit(3_i64), lit(5_i64)), + expected_greptime: "Filter: t.v BETWEEN Int16(3) AND Int16(5)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(3) AND t.v <= Int16(5)\n TableScan: t", + }, + Case { + name: "not between", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: Expr::Between(Between { + expr: Box::new(cast(col("v"), DataType::Int64)), + negated: true, + low: Box::new(lit(3_i64)), + high: Box::new(lit(5_i64)), + }), + expected_greptime: "Filter: t.v NOT BETWEEN Int16(3) AND Int16(5)\n TableScan: t", + expected_datafusion: "Filter: t.v < Int16(3) OR t.v > Int16(5)\n TableScan: t", + }, + Case { + name: "plain literal", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: col("v").gt_eq(lit(42_i64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int64(42)\n TableScan: t", + }, + Case { + name: "casted constant", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: col("v").gt_eq(cast(lit(42_i8), DataType::Int64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int64(42)\n TableScan: t", + }, + Case { + name: "timestamp downcast equality", + fields: vec![Field::new( + "ts_ns", + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + )], + predicate: cast( + col("ts_ns"), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + ) + .eq(ts_ms_literal(5000)), + expected_greptime: "Filter: CAST(t.ts_ns AS Timestamp(ms)) = TimestampMillisecond(5000, None)\n TableScan: t", + expected_datafusion: "Filter: t.ts_ns >= TimestampNanosecond(5000000000, None) AND t.ts_ns < TimestampNanosecond(5001000000, None)\n TableScan: t", + }, + Case { + name: "timestamp widening exact", + fields: vec![Field::new( + "ts_ms", + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + false, + )], + predicate: cast( + col("ts_ms"), + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + ) + .eq(lit(ScalarValue::TimestampNanosecond( + Some(5_000_000_000), + None, + ))), + expected_greptime: "Filter: CAST(t.ts_ms AS Timestamp(ns)) = TimestampNanosecond(5000000000, None)\n TableScan: t", + expected_datafusion: "Filter: t.ts_ms = TimestampMillisecond(5000, None)\n TableScan: t", + }, + ]; + + for case in cases { + let greptime = + greptime_const_normalized_filter(case.fields.clone(), case.predicate.clone()); + let datafusion = datafusion_simplified_filter(case.fields, case.predicate); + assert_eq!(case.expected_greptime, greptime, "{} greptime", case.name); + assert_eq!( + case.expected_datafusion, datafusion, + "{} datafusion", + case.name + ); + } + } + fn assert_pattern_match_plan(kind: PatternMatchKind, pattern: ScalarValue, expected: &str) { let predicate = match kind { PatternMatchKind::Like => Expr::Like(Like::new( @@ -1033,6 +1155,18 @@ mod tests { analyze_plan(build_filter_plan(test_schema(fields), predicate)) } + fn greptime_const_normalized_filter(fields: Vec, predicate: Expr) -> String { + analyze_filter(fields, predicate).to_string() + } + + fn datafusion_simplified_filter(fields: Vec, predicate: Expr) -> String { + let plan = build_filter_plan(test_schema(fields), predicate); + Optimizer::with_rules(vec![Arc::new(SimplifyExpressions::new())]) + .optimize(plan, &OptimizerContext::new(), |_, _| {}) + .unwrap() + .to_string() + } + fn analyze_plan(plan: LogicalPlan) -> LogicalPlan { ConstNormalizationRule .analyze(plan, &ConfigOptions::default()) diff --git a/tests/cases/distributed/explain/subqueries.result b/tests/cases/distributed/explain/subqueries.result index cabc20f0a6c..49bf5d97566 100644 --- a/tests/cases/distributed/explain/subqueries.result +++ b/tests/cases/distributed/explain/subqueries.result @@ -91,7 +91,7 @@ order by t.i desc; |_|_Cross Join:_| |_|_Projection: integers.i_| |_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: integers.i IS NOT NULL AND Boolean(true)_| +|_| Filter: integers.i IS NOT NULL_| |_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL, Boolean(true)] | |_| ]]_| |_|_Projection:_| diff --git a/tests/cases/standalone/common/filter/cast_preimage.result b/tests/cases/standalone/common/filter/cast_preimage.result new file mode 100644 index 00000000000..3209a842e04 --- /dev/null +++ b/tests/cases/standalone/common/filter/cast_preimage.result @@ -0,0 +1,394 @@ +-- SQL cast predicate preimage / const-normalization filter pushdown. +-- This is deliberately plain SQL, not PromQL/TQL, so it covers the generic +-- planner path where cast predicates should become scan-local filters. +CREATE TABLE cast_preimage_ts ( + host STRING PRIMARY KEY, + ts TIMESTAMP(9) TIME INDEX, + v INTEGER, +); + +Affected Rows: 0 + +INSERT INTO cast_preimage_ts VALUES + ('host1', 0, 1), + ('host1', 5000000000, 2), + ('host1', 10000000000, 3), + ('host2', 15000000000, 4); + +Affected Rows: 4 + +-- Timestamp downcast comparison: +-- CAST(ts_ns AS TIMESTAMP(3)) >= TIMESTAMP(3) '1970-01-01 00:00:05' +-- should be pushed as a native nanosecond bound on the scan. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) >= '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------+ + +-- Timestamp downcast BETWEEN should be lowered through DataFusion's BETWEEN +-- simplification and cast preimage, not left as a CAST predicate in scan filters. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) BETWEEN '1970-01-01 00:00:00'::TIMESTAMP(3) + AND '1970-01-01 00:00:10'::TIMESTAMP(3) +ORDER BY host, v; + ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(-999999, None) AND cast_preimage_ts.ts < TimestampNanosecond(10001000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(-999999, None), cast_preimage_ts.ts < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Timestamp downcast equality should become a native half-open nanosecond range. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) = '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None) AND cast_preimage_ts.ts < TimestampNanosecond(5001000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None), cast_preimage_ts.ts < TimestampNanosecond(5001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Literal-left timestamp inequality should normalize before cast preimage. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE '1970-01-01 00:00:05'::TIMESTAMP(3) < CAST(ts AS TIMESTAMP(3)) +ORDER BY host, v; + ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(5001000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(5001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------+ + +-- Integer widening comparison and IN-list cover the overlap with the old +-- Greptime ConstNormalizationRule's lossless-cast cases. +CREATE TABLE cast_preimage_int ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + v SMALLINT, +); + +Affected Rows: 0 + +INSERT INTO cast_preimage_int VALUES + (0, 'host1', 1), + (1000, 'host2', 2), + (2000, 'host3', 3); + +Affected Rows: 3 + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) >= 2 +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) IN (1, 3) +ORDER BY host; + ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v = Int16(1) OR cast_preimage_int.v = Int16(3) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v = Int16(1) OR cast_preimage_int.v = Int16(3)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------+ + +-- Swapped comparison and BETWEEN/NOT BETWEEN cover const-normalization overlap. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE 2 <= CAST(v AS BIGINT) +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) BETWEEN 1 AND 2 +ORDER BY host; + ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(1) AND cast_preimage_int.v <= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(1), cast_preimage_int.v <= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) NOT BETWEEN 1 AND 2 +ORDER BY host; + ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v < Int16(1) OR cast_preimage_int.v > Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v < Int16(1) OR cast_preimage_int.v > Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------+ + +-- Plain and casted constants should become scan-local typed literals. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= 2 +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= CAST(2 AS BIGINT) +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- TRY_CAST widening should also simplify to a scan-local typed literal. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE TRY_CAST(v AS BIGINT) >= 2 +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- Unsafe narrowing and int-to-string roundtrip stay result-correct. +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS TINYINT) AS SMALLINT) = v +ORDER BY host; + ++-------+---+ +| host | v | ++-------+---+ +| host1 | 1 | +| host2 | 2 | +| host3 | 3 | ++-------+---+ + +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS STRING) AS SMALLINT) = v +ORDER BY host; + ++-------+---+ +| host | v | ++-------+---+ +| host1 | 1 | +| host2 | 2 | +| host3 | 3 | ++-------+---+ + +CREATE TABLE cast_preimage_ts_ms ( + host STRING PRIMARY KEY, + ts TIMESTAMP(3) TIME INDEX, + v INTEGER, +); + +Affected Rows: 0 + +INSERT INTO cast_preimage_ts_ms VALUES + ('host1', 0, 1), + ('host2', 5000, 2), + ('host3', 5001, 3); + +Affected Rows: 3 + +-- Timestamp widening equality is exact at millisecond precision. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05'::TIMESTAMP(9) +ORDER BY host; + ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts_ms.host ASC NULLS LAST | +| | Projection: cast_preimage_ts_ms.host, cast_preimage_ts_ms.v | +| | Filter: cast_preimage_ts_ms.ts = TimestampMillisecond(5000, None) | +| | TableScan: cast_preimage_ts_ms, partial_filters=[cast_preimage_ts_ms.ts = TimestampMillisecond(5000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------+ + +-- Non-exact nanosecond literal should remain semantically correct. +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05.000000001'::TIMESTAMP(9) +ORDER BY host; + +++ +++ + +DROP TABLE cast_preimage_ts; + +Affected Rows: 0 + +DROP TABLE cast_preimage_int; + +Affected Rows: 0 + +DROP TABLE cast_preimage_ts_ms; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/filter/cast_preimage.sql b/tests/cases/standalone/common/filter/cast_preimage.sql new file mode 100644 index 00000000000..7ee7e8198f3 --- /dev/null +++ b/tests/cases/standalone/common/filter/cast_preimage.sql @@ -0,0 +1,175 @@ +-- SQL cast predicate preimage / const-normalization filter pushdown. +-- This is deliberately plain SQL, not PromQL/TQL, so it covers the generic +-- planner path where cast predicates should become scan-local filters. + +CREATE TABLE cast_preimage_ts ( + host STRING PRIMARY KEY, + ts TIMESTAMP(9) TIME INDEX, + v INTEGER, +); + +INSERT INTO cast_preimage_ts VALUES + ('host1', 0, 1), + ('host1', 5000000000, 2), + ('host1', 10000000000, 3), + ('host2', 15000000000, 4); + +-- Timestamp downcast comparison: +-- CAST(ts_ns AS TIMESTAMP(3)) >= TIMESTAMP(3) '1970-01-01 00:00:05' +-- should be pushed as a native nanosecond bound on the scan. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) >= '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + +-- Timestamp downcast BETWEEN should be lowered through DataFusion's BETWEEN +-- simplification and cast preimage, not left as a CAST predicate in scan filters. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) BETWEEN '1970-01-01 00:00:00'::TIMESTAMP(3) + AND '1970-01-01 00:00:10'::TIMESTAMP(3) +ORDER BY host, v; + +-- Timestamp downcast equality should become a native half-open nanosecond range. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) = '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + +-- Literal-left timestamp inequality should normalize before cast preimage. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE '1970-01-01 00:00:05'::TIMESTAMP(3) < CAST(ts AS TIMESTAMP(3)) +ORDER BY host, v; + +-- Integer widening comparison and IN-list cover the overlap with the old +-- Greptime ConstNormalizationRule's lossless-cast cases. +CREATE TABLE cast_preimage_int ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + v SMALLINT, +); + +INSERT INTO cast_preimage_int VALUES + (0, 'host1', 1), + (1000, 'host2', 2), + (2000, 'host3', 3); + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) >= 2 +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) IN (1, 3) +ORDER BY host; + +-- Swapped comparison and BETWEEN/NOT BETWEEN cover const-normalization overlap. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE 2 <= CAST(v AS BIGINT) +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) BETWEEN 1 AND 2 +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) NOT BETWEEN 1 AND 2 +ORDER BY host; + +-- Plain and casted constants should become scan-local typed literals. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= 2 +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= CAST(2 AS BIGINT) +ORDER BY host; + +-- TRY_CAST widening should also simplify to a scan-local typed literal. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE TRY_CAST(v AS BIGINT) >= 2 +ORDER BY host; + +-- Unsafe narrowing and int-to-string roundtrip stay result-correct. +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS TINYINT) AS SMALLINT) = v +ORDER BY host; + +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS STRING) AS SMALLINT) = v +ORDER BY host; + +CREATE TABLE cast_preimage_ts_ms ( + host STRING PRIMARY KEY, + ts TIMESTAMP(3) TIME INDEX, + v INTEGER, +); + +INSERT INTO cast_preimage_ts_ms VALUES + ('host1', 0, 1), + ('host2', 5000, 2), + ('host3', 5001, 3); + +-- Timestamp widening equality is exact at millisecond precision. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05'::TIMESTAMP(9) +ORDER BY host; + +-- Non-exact nanosecond literal should remain semantically correct. +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05.000000001'::TIMESTAMP(9) +ORDER BY host; + +DROP TABLE cast_preimage_ts; + +DROP TABLE cast_preimage_int; + +DROP TABLE cast_preimage_ts_ms; diff --git a/tests/cases/standalone/common/promql/precisions.result b/tests/cases/standalone/common/promql/precisions.result index ef67f1aec78..e57f5b04ee4 100644 --- a/tests/cases/standalone/common/promql/precisions.result +++ b/tests/cases/standalone/common/promql/precisions.result @@ -38,6 +38,26 @@ INSERT INTO TABLE host_micro VALUES Affected Rows: 8 +CREATE TABLE host_nano ( + ts timestamp(9) time index, + host STRING PRIMARY KEY, + val DOUBLE, +); + +Affected Rows: 0 + +INSERT INTO TABLE host_nano VALUES + (0, 'host1', 1), + (0, 'host2', 2), + (5000000000, 'host1', 3), + (5000000000, 'host2', 4), + (10000000000, 'host1', 5), + (10000000000, 'host2', 6), + (15000000000, 'host1', 7), + (15000000000, 'host2', 8); + +Affected Rows: 8 + -- Test on Timestamps of different precisions -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') host_sec{host="host1"}; @@ -111,6 +131,118 @@ TQL EVAL (0, 15, '5s') avg_over_time(host_sec{host="host1"}[5s]) + avg_over_time | host1 | 1970-01-01T00:00:15 | 14.0 | +-------+---------------------+-----------------------------------------------------------------------------------------+ +-- Verify that PromQL time predicates on non-millisecond time indexes are +-- pushed into the scan as native timestamp range filters. +-- Original instant selector filter is built on the millisecond alias: +-- host = "host1" AND ts_ms >= -299999ms AND ts_ms <= 10000ms +-- After pushing through `CAST(raw_ts AS Timestamp(ms)) AS ts` and applying +-- DataFusion cast preimage, it becomes a native half-open range on raw_ts. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_micro{host="host1"}; + ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_micro.host ASC NULLS FIRST, host_micro.ts ASC NULLS FIRST | +| | Projection: host_micro.val, host_micro.host, CAST(host_micro.ts AS Timestamp(ms)) AS ts | +| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts >= TimestampMicrosecond(-299999999, None) AND host_micro.ts < TimestampMicrosecond(10001000, None) | +| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts >= TimestampMicrosecond(-299999999, None), host_micro.ts < TimestampMicrosecond(10001000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- The same instant-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -299999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; + ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_nano.host ASC NULLS FIRST, host_nano.ts ASC NULLS FIRST | +| | Projection: host_nano.val, host_nano.host, CAST(host_nano.ts AS Timestamp(ms)) AS ts | +| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts >= TimestampNanosecond(-299999999999, None) AND host_nano.ts < TimestampNanosecond(10001000000, None) | +| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts >= TimestampNanosecond(-299999999999, None), host_nano.ts < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Range selectors use their range window instead of the default lookback. +-- Original range selector filter for [5s]: +-- host = "host1" AND ts_ms >= -4999ms AND ts_ms <= 10000ms +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_micro{host="host1"}[5s]); + ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Filter: prom_avg_over_time(ts_range,val) IS NOT NULL | +| | Projection: host_micro.ts, prom_avg_over_time(ts_range, val) AS prom_avg_over_time(ts_range,val), host_micro.host | +| | PromRangeManipulate: req range=[0..10000], interval=[5000], eval range=[5000], time index=[ts], values=["val"] | +| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_micro.host ASC NULLS FIRST, host_micro.ts ASC NULLS FIRST | +| | Projection: host_micro.val, host_micro.host, CAST(host_micro.ts AS Timestamp(ms)) AS ts | +| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts >= TimestampMicrosecond(-4999999, None) AND host_micro.ts < TimestampMicrosecond(10001000, None) | +| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts >= TimestampMicrosecond(-4999999, None), host_micro.ts < TimestampMicrosecond(10001000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- The same range-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -4999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_nano{host="host1"}[5s]); + ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Filter: prom_avg_over_time(ts_range,val) IS NOT NULL | +| | Projection: host_nano.ts, prom_avg_over_time(ts_range, val) AS prom_avg_over_time(ts_range,val), host_nano.host | +| | PromRangeManipulate: req range=[0..10000], interval=[5000], eval range=[5000], time index=[ts], values=["val"] | +| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_nano.host ASC NULLS FIRST, host_nano.ts ASC NULLS FIRST | +| | Projection: host_nano.val, host_nano.host, CAST(host_nano.ts AS Timestamp(ms)) AS ts | +| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts >= TimestampNanosecond(-4999999999, None) AND host_nano.ts < TimestampNanosecond(10001000000, None) | +| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts >= TimestampNanosecond(-4999999999, None), host_nano.ts < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + DROP TABLE host_sec; Affected Rows: 0 @@ -119,3 +251,7 @@ DROP TABLE host_micro; Affected Rows: 0 +DROP TABLE host_nano; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/precisions.sql b/tests/cases/standalone/common/promql/precisions.sql index 6966e5bca20..01e6cae9fec 100644 --- a/tests/cases/standalone/common/promql/precisions.sql +++ b/tests/cases/standalone/common/promql/precisions.sql @@ -30,6 +30,22 @@ INSERT INTO TABLE host_micro VALUES (15000000, 'host1', 7), (15000000, 'host2', 8); +CREATE TABLE host_nano ( + ts timestamp(9) time index, + host STRING PRIMARY KEY, + val DOUBLE, +); + +INSERT INTO TABLE host_nano VALUES + (0, 'host1', 1), + (0, 'host2', 2), + (5000000000, 'host1', 3), + (5000000000, 'host2', 4), + (10000000000, 'host1', 5), + (10000000000, 'host2', 6), + (15000000000, 'host1', 7), + (15000000000, 'host2', 8); + -- Test on Timestamps of different precisions -- SQLNESS SORT_RESULT 3 1 @@ -50,6 +66,50 @@ TQL EVAL (0, 15, '5s') host_sec{host="host1"} + host_micro{host="host1"}; -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') avg_over_time(host_sec{host="host1"}[5s]) + avg_over_time(host_micro{host="host1"}[5s]); +-- Verify that PromQL time predicates on non-millisecond time indexes are +-- pushed into the scan as native timestamp range filters. +-- Original instant selector filter is built on the millisecond alias: +-- host = "host1" AND ts_ms >= -299999ms AND ts_ms <= 10000ms +-- After pushing through `CAST(raw_ts AS Timestamp(ms)) AS ts` and applying +-- DataFusion cast preimage, it becomes a native half-open range on raw_ts. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_micro{host="host1"}; + +-- The same instant-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -299999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; + +-- Range selectors use their range window instead of the default lookback. +-- Original range selector filter for [5s]: +-- host = "host1" AND ts_ms >= -4999ms AND ts_ms <= 10000ms +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_micro{host="host1"}[5s]); + +-- The same range-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -4999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_nano{host="host1"}[5s]); + DROP TABLE host_sec; DROP TABLE host_micro; + +DROP TABLE host_nano; diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 810dd276442..e6c9eaec965 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -380,25 +380,21 @@ Affected Rows: 3 -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED TQL EXPLAIN (0, 10, '5s') test_nano; -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | -| | PromSeriesDivide: tags=["k"] | -| | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST | -| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None) | -| | TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | -| | ]] | -| physical_plan | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | -| | PromSeriesDivideExec: tags=["k"] | -| | SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true] | -| | RepartitionExec: REDACTED -| | ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j] | -| | MergeScanExec: REDACTED -| | | -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | +| | PromSeriesDivide: tags=["k"] | +| | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST | +| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j | +| | Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None) | +| | TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- explain verbose at 0s, 5s and 10s for a nanosecond time index. -- SQLNESS REPLACE (-+) - @@ -425,14 +421,14 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | logical_plan after resolve_grouping_function_| SAME TEXT AS ABOVE_| | logical_plan after type_coercion_| SAME TEXT AS ABOVE_| | logical_plan after ConstNormalizationRule_| SAME TEXT AS ABOVE_| -| logical_plan after DistPlannerAnalyzer_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +| logical_plan after DistPlannerAnalyzer_| Projection: test_nano.i, test_nano.k, test_nano.j_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| |_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| -|_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| -|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | +|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | |_| ]]_| | logical_plan after FixStateUdafOrderingAnalyzer_| SAME TEXT AS ABOVE_| | analyzed_logical_plan_| SAME TEXT AS ABOVE_| @@ -459,13 +455,13 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_| | logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_| | logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_| -| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +| logical_plan after optimize_projections_| MergeScan [is_placeholder=false, remote_input=[_| +|_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| |_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| -|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | +|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | |_| ]]_| | logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| | logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| @@ -495,37 +491,21 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | logical_plan after optimize_projections_| SAME TEXT AS ABOVE_| | logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| | logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| -| logical_plan_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +| logical_plan_| MergeScan [is_placeholder=false, remote_input=[_| +|_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| |_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| -|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | +|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | |_| ]]_| -| initial_physical_plan_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED +| initial_physical_plan_| MergeScanExec: REDACTED |_|_| -| initial_physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_MergeScanExec: REDACTED +| initial_physical_plan_with_stats_| MergeScanExec: REDACTED |_|_| -| initial_physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_PromSeriesDivideExec: tags=["k"], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_MergeScanExec: REDACTED +| initial_physical_plan_with_schema_| MergeScanExec: REDACTED |_|_| | physical_plan after OutputRequirements_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| |_|_MergeScanExec: REDACTED |_|_| | physical_plan after aggregate_statistics_| SAME TEXT AS ABOVE_| @@ -535,70 +515,34 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | physical_plan after parallelize_scan_| SAME TEXT AS ABOVE_| | physical_plan after PassDistributionRule_| SAME TEXT AS ABOVE_| | physical_plan after PromqlTsidNarrowJoin_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED -|_|_| -| physical_plan after EnforceDistribution_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED -|_|_| +| physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| +| physical_plan after EnforceDistribution_| SAME TEXT AS ABOVE_| | physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED -|_|_| +| physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| | physical_plan after OptimizeAggregateOrder_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| -| physical_plan after OutputRequirements_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED +| physical_plan after OutputRequirements_| MergeScanExec: REDACTED |_|_| | physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushPastWindows_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushdown_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after PushdownSort_| SAME TEXT AS ABOVE_| -| physical_plan after EnsureCooperative_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureCooperative_| CooperativeExec_| +|_|_MergeScanExec: REDACTED +|_|_| | physical_plan after FilterPushdown(Post)_| SAME TEXT AS ABOVE_| | physical_plan after WindowedSortRule_| SAME TEXT AS ABOVE_| | physical_plan after MatchesConstantTerm_| SAME TEXT AS ABOVE_| | physical_plan after RemoveDuplicateRule_| SAME TEXT AS ABOVE_| | physical_plan after SanityCheckPlan_| SAME TEXT AS ABOVE_| -| physical_plan_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +| physical_plan_| CooperativeExec_| |_|_MergeScanExec: REDACTED |_|_| -| physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +| physical_plan_with_stats_| CooperativeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| |_|_MergeScanExec: REDACTED |_|_| -| physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_PromSeriesDivideExec: tags=["k"], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +| physical_plan_with_schema_| CooperativeExec, schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ diff --git a/tests/cases/standalone/common/tql/general_table.result b/tests/cases/standalone/common/tql/general_table.result index 949c1f8d716..6172f40aaad 100644 --- a/tests/cases/standalone/common/tql/general_table.result +++ b/tests/cases/standalone/common/tql/general_table.result @@ -18,6 +18,10 @@ Affected Rows: 0 -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range +-- SQLNESS REPLACE (cpu_usage\.ts) ts +-- SQLNESS REPLACE (?m)^\|_\|_\|_SortExec:.*\n\|_\|_\|_RepartitionExec:.*\n +-- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; @@ -25,7 +29,10 @@ TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[job@0 as job, ts@1 as ts, sum(prom_irate(ts_range,value))@2 / 1000000000 as sum(prom_irate(ts_range,value)) / Float64(1000000000)] REDACTED +| 0_| 0_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_ProjectionExec: expr=[job@0 as job, ts@1 as ts, sum(prom_irate(ts_range,value))@2 / 1000000000 as sum(prom_irate(ts_range,value)) / Float64(1000000000)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_SortPreservingMergeExec: [job@0 ASC NULLS LAST, ts@1 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[job@0 ASC NULLS LAST, ts@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED @@ -35,13 +42,8 @@ TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[5000], time index=[ts] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["job"] REDACTED -|_|_|_SortExec: expr=[job@1 ASC, ts@2 ASC], preserve_partitioning=[true] REDACTED -|_|_|_RepartitionExec: partitioning=REDACTED |_|_|_ProjectionExec: expr=[value@1 as value, job@0 as job, CAST(ts@2 AS Timestamp(ms)) as ts] REDACTED -|_|_|_MergeScanExec: REDACTED -|_|_|_| -| 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_ScanExec: REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/common/tql/general_table.sql b/tests/cases/standalone/common/tql/general_table.sql index 573d0cbdff9..6bf9549773c 100644 --- a/tests/cases/standalone/common/tql/general_table.sql +++ b/tests/cases/standalone/common/tql/general_table.sql @@ -16,6 +16,10 @@ WITH( -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range +-- SQLNESS REPLACE (cpu_usage\.ts) ts +-- SQLNESS REPLACE (?m)^\|_\|_\|_SortExec:.*\n\|_\|_\|_RepartitionExec:.*\n +-- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9;